semantic_logger 4.17.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +42 -81
- data/Rakefile +8 -1
- data/lib/semantic_logger/appender/async.rb +86 -173
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +4 -4
- data/lib/semantic_logger/appender/elasticsearch.rb +6 -182
- data/lib/semantic_logger/appender/elasticsearch_base.rb +212 -0
- data/lib/semantic_logger/appender/elasticsearch_http.rb +2 -2
- data/lib/semantic_logger/appender/file.rb +20 -4
- data/lib/semantic_logger/appender/graylog.rb +2 -2
- data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
- data/lib/semantic_logger/appender/http.rb +27 -2
- data/lib/semantic_logger/appender/io.rb +8 -4
- data/lib/semantic_logger/appender/kafka.rb +2 -2
- data/lib/semantic_logger/appender/loki.rb +2 -4
- data/lib/semantic_logger/appender/mongodb.rb +3 -6
- data/lib/semantic_logger/appender/new_relic_logs.rb +12 -2
- data/lib/semantic_logger/appender/open_telemetry.rb +25 -11
- data/lib/semantic_logger/appender/opensearch.rb +35 -0
- data/lib/semantic_logger/appender/rabbitmq.rb +3 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +8 -3
- data/lib/semantic_logger/appender/splunk.rb +2 -2
- data/lib/semantic_logger/appender/splunk_http.rb +3 -3
- data/lib/semantic_logger/appender/syslog.rb +5 -4
- data/lib/semantic_logger/appender/tcp.rb +2 -2
- data/lib/semantic_logger/appender/udp.rb +2 -2
- data/lib/semantic_logger/appender/wrapper.rb +4 -4
- data/lib/semantic_logger/appender.rb +30 -19
- data/lib/semantic_logger/appenders.rb +26 -5
- data/lib/semantic_logger/base.rb +113 -21
- data/lib/semantic_logger/concerns/compatibility.rb +2 -2
- data/lib/semantic_logger/core_ext/process.rb +34 -0
- data/lib/semantic_logger/formatters/base.rb +46 -7
- data/lib/semantic_logger/formatters/color.rb +6 -3
- data/lib/semantic_logger/formatters/default.rb +6 -4
- data/lib/semantic_logger/formatters/ecs.rb +151 -0
- data/lib/semantic_logger/formatters/fluentd.rb +15 -4
- data/lib/semantic_logger/formatters/json.rb +6 -1
- data/lib/semantic_logger/formatters/logfmt.rb +2 -2
- data/lib/semantic_logger/formatters/loki.rb +4 -4
- data/lib/semantic_logger/formatters/new_relic_logs.rb +4 -6
- data/lib/semantic_logger/formatters/open_telemetry.rb +40 -2
- data/lib/semantic_logger/formatters/pattern.rb +235 -0
- data/lib/semantic_logger/formatters/raw.rb +2 -2
- data/lib/semantic_logger/formatters/signalfx.rb +2 -2
- data/lib/semantic_logger/formatters/syslog.rb +14 -3
- data/lib/semantic_logger/formatters/syslog_cee.rb +1 -1
- data/lib/semantic_logger/formatters.rb +2 -0
- data/lib/semantic_logger/log.rb +23 -4
- data/lib/semantic_logger/logger.rb +2 -2
- data/lib/semantic_logger/metric/new_relic.rb +2 -2
- data/lib/semantic_logger/metric/signalfx.rb +2 -2
- data/lib/semantic_logger/metric/statsd.rb +2 -2
- data/lib/semantic_logger/processor.rb +22 -1
- data/lib/semantic_logger/queue_processor.rb +369 -0
- data/lib/semantic_logger/reporters/minitest.rb +4 -4
- data/lib/semantic_logger/semantic_logger.rb +103 -11
- data/lib/semantic_logger/subscriber.rb +15 -2
- data/lib/semantic_logger/sync_processor.rb +25 -3
- data/lib/semantic_logger/test/capture_log_events.rb +2 -2
- data/lib/semantic_logger/test/minitest.rb +8 -4
- data/lib/semantic_logger/test/rspec.rb +249 -0
- data/lib/semantic_logger/utils.rb +83 -4
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +9 -0
- metadata +15 -6
- data/lib/semantic_logger/appender/async_batch.rb +0 -93
data/lib/semantic_logger/base.rb
CHANGED
|
@@ -34,6 +34,19 @@ module SemanticLogger
|
|
|
34
34
|
@level || SemanticLogger.default_level
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
# Set the logging level for this logger during the execution of the given block
|
|
38
|
+
#
|
|
39
|
+
# Refer to the documentation for `#level=` for more information about the possible log levels.
|
|
40
|
+
def with_level(new_level)
|
|
41
|
+
old_level = level
|
|
42
|
+
|
|
43
|
+
self.level = new_level
|
|
44
|
+
|
|
45
|
+
yield
|
|
46
|
+
ensure
|
|
47
|
+
self.level = old_level
|
|
48
|
+
end
|
|
49
|
+
|
|
37
50
|
# Implement the log level calls
|
|
38
51
|
# logger.debug(message, hash|exception=nil, &block)
|
|
39
52
|
#
|
|
@@ -185,7 +198,18 @@ module SemanticLogger
|
|
|
185
198
|
# to:
|
|
186
199
|
# `logger.tagged('first', 'more', 'other')`
|
|
187
200
|
# - For better performance with clean tags, see `SemanticLogger.tagged`.
|
|
201
|
+
#
|
|
202
|
+
# Child logger:
|
|
203
|
+
# - When called *without* a block, returns a new logger instance that permanently
|
|
204
|
+
# carries the supplied tags ("instance tags"). Those tags are added to every log
|
|
205
|
+
# entry emitted by the returned logger, and _only_ that logger, even across threads.
|
|
206
|
+
# Unlike the block form they are not pushed onto the thread, so other loggers are
|
|
207
|
+
# unaffected. This is the recommended way to bind a logger to an object's identity:
|
|
208
|
+
# logger = SemanticLogger['Cart'].tagged(cart_id: cart.id)
|
|
188
209
|
def tagged(*tags)
|
|
210
|
+
# No block: build a child logger that carries these tags on every entry.
|
|
211
|
+
return tagged_child(tags) unless block_given?
|
|
212
|
+
|
|
189
213
|
block = -> { yield(self) }
|
|
190
214
|
# Allow named tags to be passed into the logger
|
|
191
215
|
# Rails::Rack::Logger passes logs as an array with a single argument
|
|
@@ -213,6 +237,10 @@ module SemanticLogger
|
|
|
213
237
|
SemanticLogger.named_tags
|
|
214
238
|
end
|
|
215
239
|
|
|
240
|
+
# Tags permanently bound to this logger instance via `#tagged` (without a block).
|
|
241
|
+
# Empty for a regular (non-child) logger.
|
|
242
|
+
attr_reader :instance_tags, :instance_named_tags
|
|
243
|
+
|
|
216
244
|
# Returns the list of tags pushed after flattening them out and removing blanks
|
|
217
245
|
#
|
|
218
246
|
# Note:
|
|
@@ -231,13 +259,13 @@ module SemanticLogger
|
|
|
231
259
|
end
|
|
232
260
|
|
|
233
261
|
# :nodoc:
|
|
234
|
-
def silence(new_level = :error, &
|
|
235
|
-
SemanticLogger.silence(new_level, &
|
|
262
|
+
def silence(new_level = :error, &)
|
|
263
|
+
SemanticLogger.silence(new_level, &)
|
|
236
264
|
end
|
|
237
265
|
|
|
238
266
|
# :nodoc:
|
|
239
|
-
def fast_tag(tag, &
|
|
240
|
-
SemanticLogger.fast_tag(tag, &
|
|
267
|
+
def fast_tag(tag, &)
|
|
268
|
+
SemanticLogger.fast_tag(tag, &)
|
|
241
269
|
end
|
|
242
270
|
|
|
243
271
|
# Write log data to underlying data storage
|
|
@@ -250,6 +278,11 @@ module SemanticLogger
|
|
|
250
278
|
meets_log_level?(log) && !filtered?(log)
|
|
251
279
|
end
|
|
252
280
|
|
|
281
|
+
protected
|
|
282
|
+
|
|
283
|
+
# Only assigned to when building a child logger via `#tagged` (see #tagged_child).
|
|
284
|
+
attr_writer :instance_tags, :instance_named_tags
|
|
285
|
+
|
|
253
286
|
private
|
|
254
287
|
|
|
255
288
|
# Initializer for Abstract Class SemanticLogger::Base
|
|
@@ -281,8 +314,13 @@ module SemanticLogger
|
|
|
281
314
|
raise ":filter must be a Regexp, Proc, or implement :call"
|
|
282
315
|
end
|
|
283
316
|
|
|
284
|
-
@filter
|
|
285
|
-
@name
|
|
317
|
+
@filter = filter.is_a?(Regexp) ? filter.freeze : filter
|
|
318
|
+
@name = klass.is_a?(String) ? klass : klass.name
|
|
319
|
+
# `[].freeze` / `{}.freeze` compile to `opt_ary_freeze` / `opt_hash_freeze`, which return
|
|
320
|
+
# a shared frozen singleton. Every logger points at the same two objects, so these default
|
|
321
|
+
# (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
|
|
322
|
+
@instance_tags = [].freeze
|
|
323
|
+
@instance_named_tags = {}.freeze
|
|
286
324
|
if level.nil?
|
|
287
325
|
# Allow the global default level to determine this loggers log level
|
|
288
326
|
@level_index = nil
|
|
@@ -311,13 +349,59 @@ module SemanticLogger
|
|
|
311
349
|
(level_index <= (log.level_index || 0))
|
|
312
350
|
end
|
|
313
351
|
|
|
352
|
+
# Build a new Log entry, layering this logger's instance tags on top of the
|
|
353
|
+
# current thread context.
|
|
354
|
+
def new_log(level, index = nil)
|
|
355
|
+
apply_instance_tags(Log.new(name, level, index))
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Merge this logger's instance tags into the supplied Log entry.
|
|
359
|
+
#
|
|
360
|
+
# Composition (mirrors the per-logger merge rule proposed in PR #301):
|
|
361
|
+
# - Positional: thread tags first, then this logger's instance tags appended.
|
|
362
|
+
# - Named: instance named tags merged on top of the thread named tags, so on a
|
|
363
|
+
# key conflict the logger's own identity wins (it is the more specific value).
|
|
364
|
+
# Only this logger's entries are affected; the thread context is left untouched.
|
|
365
|
+
def apply_instance_tags(log)
|
|
366
|
+
unless @instance_tags.empty?
|
|
367
|
+
log.tags = log.tags.empty? ? @instance_tags.dup : log.tags + @instance_tags
|
|
368
|
+
end
|
|
369
|
+
unless @instance_named_tags.empty?
|
|
370
|
+
log.named_tags = log.named_tags.empty? ? @instance_named_tags.dup : log.named_tags.merge(@instance_named_tags)
|
|
371
|
+
end
|
|
372
|
+
log
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# Returns a copy of this logger that permanently carries the supplied tags.
|
|
376
|
+
# Positional (string) and named (Hash) tags may be mixed; they are merged on
|
|
377
|
+
# top of any instance tags this logger already carries.
|
|
378
|
+
def tagged_child(tags)
|
|
379
|
+
positional = @instance_tags.dup
|
|
380
|
+
named = @instance_named_tags.dup
|
|
381
|
+
tags.flatten.each do |tag|
|
|
382
|
+
case tag
|
|
383
|
+
when Hash
|
|
384
|
+
named = named.merge(tag)
|
|
385
|
+
when nil, ""
|
|
386
|
+
# Ignore blanks for Rails compatibility, matching the block form.
|
|
387
|
+
else
|
|
388
|
+
positional << tag.to_s
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
child = clone
|
|
393
|
+
child.instance_tags = positional.freeze
|
|
394
|
+
child.instance_named_tags = named.freeze
|
|
395
|
+
child
|
|
396
|
+
end
|
|
397
|
+
|
|
314
398
|
# Log message at the specified level
|
|
315
|
-
def log_internal(level, index, message = nil, payload = nil, exception = nil)
|
|
399
|
+
def log_internal(level, index, message = nil, payload = nil, exception = nil, &block)
|
|
316
400
|
# Handle variable number of arguments by detecting exception object and payload hash.
|
|
317
401
|
if exception.nil? && payload.nil? && message.respond_to?(:backtrace) && message.respond_to?(:message)
|
|
318
402
|
exception = message
|
|
319
403
|
message = nil
|
|
320
|
-
elsif exception.nil? && payload
|
|
404
|
+
elsif exception.nil? && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
|
|
321
405
|
exception = payload
|
|
322
406
|
payload = nil
|
|
323
407
|
elsif payload && !payload.is_a?(Hash)
|
|
@@ -325,12 +409,12 @@ module SemanticLogger
|
|
|
325
409
|
payload = nil
|
|
326
410
|
end
|
|
327
411
|
|
|
328
|
-
log =
|
|
412
|
+
log = new_log(level, index)
|
|
329
413
|
should_log =
|
|
330
414
|
if exception.nil? && payload.nil? && message.is_a?(Hash)
|
|
331
415
|
# All arguments as a hash in the message.
|
|
332
416
|
log.assign(**log.extract_arguments(message))
|
|
333
|
-
elsif exception.nil? && message && payload
|
|
417
|
+
elsif exception.nil? && message && payload.is_a?(Hash)
|
|
334
418
|
# Message supplied along with a hash with the remaining arguments.
|
|
335
419
|
log.assign(**log.extract_arguments(payload, message))
|
|
336
420
|
else
|
|
@@ -339,20 +423,28 @@ module SemanticLogger
|
|
|
339
423
|
end
|
|
340
424
|
|
|
341
425
|
# Add result of block to message or payload if not nil
|
|
342
|
-
if
|
|
343
|
-
result = yield(log)
|
|
344
|
-
case result
|
|
345
|
-
when String
|
|
346
|
-
log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
|
|
347
|
-
when Hash
|
|
348
|
-
log.assign_hash(result)
|
|
349
|
-
end
|
|
350
|
-
end
|
|
426
|
+
apply_block_result(log, &block) if block
|
|
351
427
|
|
|
352
428
|
# Log level may change during assign due to :on_exception_level
|
|
353
429
|
self.log(log) if should_log && should_log?(log)
|
|
354
430
|
end
|
|
355
431
|
|
|
432
|
+
# Merge the result of a logging block into the log entry.
|
|
433
|
+
#
|
|
434
|
+
# Zero-arity blocks (e.g. `logger.info { "msg" }` or a zero-arity lambda) are called without
|
|
435
|
+
# arguments. Lambdas enforce their arity, so yielding the log to a zero-arity lambda would
|
|
436
|
+
# raise ArgumentError. Blocks that accept an argument still receive the log so they can read
|
|
437
|
+
# or mutate it.
|
|
438
|
+
def apply_block_result(log, &block)
|
|
439
|
+
result = block.arity.zero? ? block.call : block.call(log)
|
|
440
|
+
case result
|
|
441
|
+
when String
|
|
442
|
+
log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
|
|
443
|
+
when Hash
|
|
444
|
+
log.assign_hash(result)
|
|
445
|
+
end
|
|
446
|
+
end
|
|
447
|
+
|
|
356
448
|
# Measure the supplied block and log the message
|
|
357
449
|
def measure_internal(level, index, message, params)
|
|
358
450
|
exception = nil
|
|
@@ -376,7 +468,7 @@ module SemanticLogger
|
|
|
376
468
|
exception = e
|
|
377
469
|
ensure
|
|
378
470
|
# Must use ensure block otherwise a `return` in the yield above will skip the log entry
|
|
379
|
-
log =
|
|
471
|
+
log = new_log(level, index)
|
|
380
472
|
exception ||= params[:exception]
|
|
381
473
|
message = params[:message] if params[:message]
|
|
382
474
|
duration =
|
|
@@ -424,7 +516,7 @@ module SemanticLogger
|
|
|
424
516
|
rescue Exception => e
|
|
425
517
|
exception = e
|
|
426
518
|
ensure
|
|
427
|
-
log =
|
|
519
|
+
log = new_log(level, index)
|
|
428
520
|
# May return false due to elastic logging
|
|
429
521
|
should_log = log.assign(
|
|
430
522
|
message: message,
|
|
@@ -37,11 +37,11 @@ module SemanticLogger
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# :nodoc:
|
|
40
|
-
def add(severity, message = nil, progname = nil, &
|
|
40
|
+
def add(severity, message = nil, progname = nil, &)
|
|
41
41
|
index = Levels.index(severity)
|
|
42
42
|
if level_index <= index
|
|
43
43
|
level = Levels.level(index)
|
|
44
|
-
log_internal(level, index, message, progname, &
|
|
44
|
+
log_internal(level, index, message, progname, &)
|
|
45
45
|
true
|
|
46
46
|
else
|
|
47
47
|
false
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
module CoreExt
|
|
3
|
+
# Reopen all appenders in the child process after a fork.
|
|
4
|
+
#
|
|
5
|
+
# Prepended onto `Process.singleton_class` when Semantic Logger is loaded.
|
|
6
|
+
# Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
|
|
7
|
+
#
|
|
8
|
+
# `Process._fork` (Ruby 3.1+) is the single method that every fork path routes
|
|
9
|
+
# through (`Kernel#fork`, `Process.fork`, `IO.popen`, `Kernel#system`, and
|
|
10
|
+
# backticks), so overriding it covers them all with one hook.
|
|
11
|
+
#
|
|
12
|
+
# Note: both `_fork` and `daemon` must be overridden to reliably restart the
|
|
13
|
+
# appender thread, since `Process.daemon` does not route through `_fork`
|
|
14
|
+
# (https://bugs.ruby-lang.org/issues/18911). Do not collapse this down to only
|
|
15
|
+
# `_fork`.
|
|
16
|
+
module Process
|
|
17
|
+
# `_fork` runs in both the parent and the child. It returns 0 in the child
|
|
18
|
+
# and the child's pid in the parent, so only reopen in the child. Reopening
|
|
19
|
+
# in the parent would needlessly recreate its queue and worker thread,
|
|
20
|
+
# risking the loss of messages still on the queue.
|
|
21
|
+
def _fork
|
|
22
|
+
child_pid = super
|
|
23
|
+
SemanticLogger.reopen if child_pid.zero? && SemanticLogger.reopen_on_fork?
|
|
24
|
+
child_pid
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# `Process.daemon` does not route through `_fork`, so reopen explicitly. Once
|
|
28
|
+
# it returns, the caller is running as the daemon (child) process.
|
|
29
|
+
def daemon(...)
|
|
30
|
+
super.tap { SemanticLogger.reopen if SemanticLogger.reopen_on_fork? }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -2,7 +2,21 @@ require "time"
|
|
|
2
2
|
module SemanticLogger
|
|
3
3
|
module Formatters
|
|
4
4
|
class Base
|
|
5
|
-
attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision
|
|
5
|
+
attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
|
|
6
|
+
:escape_control_chars
|
|
7
|
+
|
|
8
|
+
# Printable escapes for the most common control characters. Any other
|
|
9
|
+
# control character is escaped to its hexadecimal `\xHH` form by
|
|
10
|
+
# #escape_control_characters.
|
|
11
|
+
CONTROL_CHAR_ESCAPES = {
|
|
12
|
+
"\n" => "\\n",
|
|
13
|
+
"\r" => "\\r",
|
|
14
|
+
"\t" => "\\t",
|
|
15
|
+
"\e" => "\\e"
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
# Matches C0 control characters (including newlines and the ANSI escape) and DEL.
|
|
19
|
+
CONTROL_CHARS = /[\x00-\x1f\x7f]/
|
|
6
20
|
|
|
7
21
|
# Time precision varies by Ruby interpreter
|
|
8
22
|
# JRuby 9.1.8.0 supports microseconds
|
|
@@ -34,16 +48,27 @@ module SemanticLogger
|
|
|
34
48
|
# precision: [Integer]
|
|
35
49
|
# How many fractional digits to log times with.
|
|
36
50
|
# Default: PRECISION (6, except on older JRuby, where 3)
|
|
51
|
+
# escape_control_chars: [Boolean]
|
|
52
|
+
# Replace control characters (newlines, the ANSI escape, etc.) in
|
|
53
|
+
# untrusted log data (message, tags, named tags, and exception
|
|
54
|
+
# message) with a printable escaped form, e.g. "\n".
|
|
55
|
+
# This prevents log forging and terminal escape-sequence injection
|
|
56
|
+
# when logging data that may contain attacker-controlled content.
|
|
57
|
+
# Note: Has no effect on structured formatters such as :json, which
|
|
58
|
+
# already escape control characters via JSON encoding.
|
|
59
|
+
# Default: false (preserve newlines and ANSI colors in text output)
|
|
37
60
|
def initialize(time_format: nil,
|
|
38
61
|
log_host: true,
|
|
39
62
|
log_application: true,
|
|
40
63
|
log_environment: true,
|
|
41
|
-
precision: PRECISION
|
|
42
|
-
|
|
43
|
-
@
|
|
44
|
-
@
|
|
45
|
-
@
|
|
46
|
-
@
|
|
64
|
+
precision: PRECISION,
|
|
65
|
+
escape_control_chars: false)
|
|
66
|
+
@time_format = time_format || self.class.build_time_format(precision)
|
|
67
|
+
@log_host = log_host
|
|
68
|
+
@log_application = log_application
|
|
69
|
+
@log_environment = log_environment
|
|
70
|
+
@precision = precision
|
|
71
|
+
@escape_control_chars = escape_control_chars
|
|
47
72
|
end
|
|
48
73
|
|
|
49
74
|
# Return default time format string
|
|
@@ -68,6 +93,20 @@ module SemanticLogger
|
|
|
68
93
|
|
|
69
94
|
private
|
|
70
95
|
|
|
96
|
+
# When `escape_control_chars` is enabled, return a copy of the supplied
|
|
97
|
+
# value with any control characters replaced by a printable escaped form
|
|
98
|
+
# so that untrusted log data cannot forge log entries or inject terminal
|
|
99
|
+
# escape sequences. Otherwise the value is returned unchanged.
|
|
100
|
+
#
|
|
101
|
+
# Note: This escapes (preserves) control characters, and is opt-in. It is
|
|
102
|
+
# distinct from Log#cleansed_message, which unconditionally *strips* ANSI
|
|
103
|
+
# colorization from the message for structured (JSON/Loki) output.
|
|
104
|
+
def escape_control_characters(value)
|
|
105
|
+
return value unless escape_control_chars && value
|
|
106
|
+
|
|
107
|
+
value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
|
|
108
|
+
end
|
|
109
|
+
|
|
71
110
|
# Return the Time as a formatted string
|
|
72
111
|
def format_time(time)
|
|
73
112
|
time = time.dup
|
|
@@ -85,7 +85,10 @@ module SemanticLogger
|
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
def tags
|
|
88
|
-
|
|
88
|
+
return if log.tags.nil? || log.tags.empty?
|
|
89
|
+
|
|
90
|
+
tags = log.tags.map { |tag| escape_control_characters(tag) }
|
|
91
|
+
"[#{color}#{tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]"
|
|
89
92
|
end
|
|
90
93
|
|
|
91
94
|
# Named Tags
|
|
@@ -94,7 +97,7 @@ module SemanticLogger
|
|
|
94
97
|
return if named_tags.nil? || named_tags.empty?
|
|
95
98
|
|
|
96
99
|
list = []
|
|
97
|
-
named_tags.each_pair { |name, value| list << "#{color}#{name}: #{value}#{color_map.clear}" }
|
|
100
|
+
named_tags.each_pair { |name, value| list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}" }
|
|
98
101
|
"{#{list.join(', ')}}"
|
|
99
102
|
end
|
|
100
103
|
|
|
@@ -123,7 +126,7 @@ module SemanticLogger
|
|
|
123
126
|
def exception
|
|
124
127
|
return unless log.exception
|
|
125
128
|
|
|
126
|
-
"-- Exception: #{color}#{log.exception.class}: #{log.exception.message}#{color_map.clear}\n#{log.backtrace_to_s}"
|
|
129
|
+
"-- Exception: #{color}#{log.exception.class}: #{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
|
|
127
130
|
end
|
|
128
131
|
|
|
129
132
|
def call(log, logger)
|
|
@@ -32,7 +32,9 @@ module SemanticLogger
|
|
|
32
32
|
|
|
33
33
|
# Tags
|
|
34
34
|
def tags
|
|
35
|
-
|
|
35
|
+
return if log.tags.nil? || log.tags.empty?
|
|
36
|
+
|
|
37
|
+
"[#{log.tags.map { |tag| escape_control_characters(tag) }.join('] [')}]"
|
|
36
38
|
end
|
|
37
39
|
|
|
38
40
|
# Named Tags
|
|
@@ -41,7 +43,7 @@ module SemanticLogger
|
|
|
41
43
|
return if named_tags.nil? || named_tags.empty?
|
|
42
44
|
|
|
43
45
|
list = []
|
|
44
|
-
named_tags.each_pair { |name, value| list << "#{name}: #{value}" }
|
|
46
|
+
named_tags.each_pair { |name, value| list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }
|
|
45
47
|
"{#{list.join(', ')}}"
|
|
46
48
|
end
|
|
47
49
|
|
|
@@ -57,7 +59,7 @@ module SemanticLogger
|
|
|
57
59
|
|
|
58
60
|
# Log message
|
|
59
61
|
def message
|
|
60
|
-
"-- #{log.message}" if log.message
|
|
62
|
+
"-- #{escape_control_characters(log.message)}" if log.message
|
|
61
63
|
end
|
|
62
64
|
|
|
63
65
|
# Payload
|
|
@@ -70,7 +72,7 @@ module SemanticLogger
|
|
|
70
72
|
|
|
71
73
|
# Exception
|
|
72
74
|
def exception
|
|
73
|
-
"-- Exception: #{log.exception.class}: #{log.exception.message}\n#{log.backtrace_to_s}" if log.exception
|
|
75
|
+
"-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n#{log.backtrace_to_s}" if log.exception
|
|
74
76
|
end
|
|
75
77
|
|
|
76
78
|
# Default text log format
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
module SemanticLogger
|
|
3
|
+
module Formatters
|
|
4
|
+
# Formatter conforming to the Elastic Common Schema (ECS).
|
|
5
|
+
#
|
|
6
|
+
# Emits log events using the nested field names defined by ECS so that they
|
|
7
|
+
# integrate cleanly with Filebeat and the Elastic stack (Elasticsearch,
|
|
8
|
+
# Kibana) without requiring an ingest pipeline to rename fields.
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: :ecs)
|
|
12
|
+
#
|
|
13
|
+
# # Route the payload, metric, and other SemanticLogger-specific data into
|
|
14
|
+
# # a custom top-level namespace (default "semantic_logger"):
|
|
15
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: "my_app"}})
|
|
16
|
+
#
|
|
17
|
+
# # Or merge the payload directly into ECS `labels` instead of a namespace:
|
|
18
|
+
# SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: nil}})
|
|
19
|
+
#
|
|
20
|
+
# == Field mapping (SemanticLogger -> ECS 8.x)
|
|
21
|
+
# time -> @timestamp (ISO-8601)
|
|
22
|
+
# level -> log.level
|
|
23
|
+
# name -> log.logger
|
|
24
|
+
# file_name / line -> log.origin.file.name / log.origin.file.line
|
|
25
|
+
# message -> message
|
|
26
|
+
# thread_name -> process.thread.name
|
|
27
|
+
# pid -> process.pid
|
|
28
|
+
# host -> host.hostname
|
|
29
|
+
# application -> service.name
|
|
30
|
+
# environment -> service.environment
|
|
31
|
+
# exception -> error.type / error.message / error.stack_trace
|
|
32
|
+
# duration -> event.duration (nanoseconds, as required by ECS)
|
|
33
|
+
# tags -> tags (ECS top-level array)
|
|
34
|
+
# named_tags -> labels.* (scalar key/value pairs)
|
|
35
|
+
# payload -> <namespace>.* (or labels.* when namespace is nil)
|
|
36
|
+
# metric/metric_amount -> <namespace>.metric / <namespace>.metric_amount
|
|
37
|
+
#
|
|
38
|
+
# == Reference
|
|
39
|
+
# * https://www.elastic.co/docs/reference/ecs
|
|
40
|
+
# * https://www.elastic.co/docs/reference/ecs/ecs-custom-fields-in-ecs
|
|
41
|
+
class Ecs < Raw
|
|
42
|
+
# ECS version this formatter targets.
|
|
43
|
+
ECS_VERSION = "8.11.0".freeze
|
|
44
|
+
|
|
45
|
+
# namespace: [String|Symbol|nil]
|
|
46
|
+
# Top-level field set used to hold SemanticLogger-specific data that has
|
|
47
|
+
# no native ECS home (payload, metric, metric_amount). A proper-noun
|
|
48
|
+
# namespace is guaranteed never to collide with a current or future ECS
|
|
49
|
+
# field. Set to nil to merge the payload into ECS `labels` instead.
|
|
50
|
+
# Default: "semantic_logger"
|
|
51
|
+
attr_reader :namespace
|
|
52
|
+
|
|
53
|
+
def initialize(namespace: "semantic_logger", time_format: :iso_8601, time_key: :timestamp, **args)
|
|
54
|
+
@namespace = namespace&.to_sym
|
|
55
|
+
|
|
56
|
+
super(time_format: time_format, time_key: time_key, **args)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Returns the log event as a single line of ECS-formatted JSON, so it can
|
|
60
|
+
# be written to stdout / a file and shipped by Filebeat or Elastic Agent.
|
|
61
|
+
def call(log, logger)
|
|
62
|
+
Utils.to_json(ecs_hash(super))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Returns a batch of log events as a single JSON array.
|
|
66
|
+
def batch(logs, logger)
|
|
67
|
+
"[#{logs.map { |log| call(log, logger) }.join(',')}]"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# Remap the flat hash built by Raw#call into the nested ECS field layout.
|
|
73
|
+
def ecs_hash(hash)
|
|
74
|
+
result = base(hash)
|
|
75
|
+
result[:process] = process(hash)
|
|
76
|
+
result[:host] = {hostname: hash[:host]} if hash[:host]
|
|
77
|
+
add_service(result, hash)
|
|
78
|
+
add_origin(result, hash)
|
|
79
|
+
add_event(result, hash)
|
|
80
|
+
result[:tags] = hash[:tags] if hash[:tags]
|
|
81
|
+
add_error(result, hash)
|
|
82
|
+
add_extras(result, hash)
|
|
83
|
+
result
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def base(hash)
|
|
87
|
+
log = {level: hash[:level].to_s}
|
|
88
|
+
log[:logger] = hash[:name] if hash[:name]
|
|
89
|
+
{
|
|
90
|
+
"@timestamp": hash[:timestamp],
|
|
91
|
+
message: hash[:message],
|
|
92
|
+
ecs: {version: ECS_VERSION},
|
|
93
|
+
log: log
|
|
94
|
+
}
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def process(hash)
|
|
98
|
+
result = {pid: hash[:pid]}
|
|
99
|
+
result[:thread] = {name: hash[:thread].to_s} if hash[:thread]
|
|
100
|
+
result
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def add_service(result, hash)
|
|
104
|
+
service = {}
|
|
105
|
+
service[:name] = hash[:application] if hash[:application]
|
|
106
|
+
service[:environment] = hash[:environment] if hash[:environment]
|
|
107
|
+
result[:service] = service unless service.empty?
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def add_origin(result, hash)
|
|
111
|
+
return unless hash[:file]
|
|
112
|
+
|
|
113
|
+
result[:log][:origin] = {file: {name: hash[:file], line: hash[:line]}.compact}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# ECS event.duration is measured in nanoseconds.
|
|
117
|
+
def add_event(result, hash)
|
|
118
|
+
return unless hash[:duration_ms]
|
|
119
|
+
|
|
120
|
+
result[:event] = {duration: (hash[:duration_ms] * 1_000_000).round}
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def add_error(result, hash)
|
|
124
|
+
return unless hash[:exception]
|
|
125
|
+
|
|
126
|
+
result[:error] = {
|
|
127
|
+
type: hash[:exception][:name],
|
|
128
|
+
message: hash[:exception][:message],
|
|
129
|
+
stack_trace: Array(hash[:exception][:stack_trace]).join("\n")
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Place SemanticLogger-specific data (payload, metric) that has no native
|
|
134
|
+
# ECS home into the configured namespace, plus named_tags into labels.
|
|
135
|
+
def add_extras(result, hash)
|
|
136
|
+
labels = hash[:named_tags].is_a?(Hash) ? hash[:named_tags].dup : {}
|
|
137
|
+
|
|
138
|
+
extra = {}
|
|
139
|
+
extra[:payload] = hash[:payload] if hash[:payload]
|
|
140
|
+
extra[:metric] = hash[:metric] if hash[:metric]
|
|
141
|
+
extra[:metric_amount] = hash[:metric_amount] if hash[:metric_amount]
|
|
142
|
+
|
|
143
|
+
unless extra.empty?
|
|
144
|
+
namespace ? result[namespace] = extra : labels.merge!(extra)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
result[:labels] = labels unless labels.empty?
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
@@ -7,9 +7,9 @@ module SemanticLogger
|
|
|
7
7
|
class Fluentd < Json
|
|
8
8
|
attr_reader :need_process_info
|
|
9
9
|
|
|
10
|
-
def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
|
|
10
|
+
def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, log_host: false, **args)
|
|
11
11
|
@need_process_info = need_process_info
|
|
12
|
-
super(time_format: time_format, time_key: time_key, **args)
|
|
12
|
+
super(time_format: time_format, time_key: time_key, log_host: log_host, **args)
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
def level
|
|
@@ -17,8 +17,19 @@ module SemanticLogger
|
|
|
17
17
|
hash["severity_index"] = log.level_index
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
# Ignore process fields: pid, thread, file and line by default.
|
|
21
|
+
# These are rarely useful under Fluentd (e.g. containerized processes
|
|
22
|
+
# usually have pid 1), so they are only included when explicitly requested
|
|
23
|
+
# via `need_process_info: true`.
|
|
24
|
+
def pid
|
|
25
|
+
super if need_process_info
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def thread_name
|
|
29
|
+
super if need_process_info
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def file_name_and_line
|
|
22
33
|
super if need_process_info
|
|
23
34
|
end
|
|
24
35
|
end
|
|
@@ -9,7 +9,12 @@ module SemanticLogger
|
|
|
9
9
|
|
|
10
10
|
# Returns log messages in JSON format
|
|
11
11
|
def call(log, logger)
|
|
12
|
-
|
|
12
|
+
Utils.to_json(super)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Returns a batch of log messages as a single JSON array.
|
|
16
|
+
def batch(logs, logger)
|
|
17
|
+
"[#{logs.map { |log| call(log, logger) }.join(',')}]"
|
|
13
18
|
end
|
|
14
19
|
end
|
|
15
20
|
end
|
|
@@ -64,9 +64,9 @@ module SemanticLogger
|
|
|
64
64
|
flattened = @parsed.map do |key, value|
|
|
65
65
|
case value
|
|
66
66
|
when Hash, Array
|
|
67
|
-
"#{key}=#{value.to_s
|
|
67
|
+
"#{key}=#{Utils.to_json(value.to_s)}"
|
|
68
68
|
else
|
|
69
|
-
"#{key}=#{
|
|
69
|
+
"#{key}=#{Utils.to_json(value)}"
|
|
70
70
|
end
|
|
71
71
|
end
|
|
72
72
|
|
|
@@ -10,7 +10,7 @@ module SemanticLogger
|
|
|
10
10
|
self.logger = logger
|
|
11
11
|
self.log = log
|
|
12
12
|
|
|
13
|
-
{streams: [build_stream]}
|
|
13
|
+
Utils.to_json({streams: [build_stream]})
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
# Returns [String] a JSON batch of logs
|
|
@@ -22,7 +22,7 @@ module SemanticLogger
|
|
|
22
22
|
build_stream
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
{streams: streams}
|
|
25
|
+
Utils.to_json({streams: streams})
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
private
|
|
@@ -82,7 +82,7 @@ module SemanticLogger
|
|
|
82
82
|
|
|
83
83
|
log.context.each do |key, value|
|
|
84
84
|
serialized_value = if value.is_a?(Hash)
|
|
85
|
-
|
|
85
|
+
Utils.to_json(value)
|
|
86
86
|
else
|
|
87
87
|
value.to_s
|
|
88
88
|
end
|
|
@@ -144,7 +144,7 @@ module SemanticLogger
|
|
|
144
144
|
|
|
145
145
|
result[string_key] = case value
|
|
146
146
|
when Hash
|
|
147
|
-
|
|
147
|
+
Utils.to_json(stringify_hash(value))
|
|
148
148
|
else
|
|
149
149
|
value.to_s
|
|
150
150
|
end
|