semantic_logger 4.17.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +42 -81
  3. data/Rakefile +8 -1
  4. data/lib/semantic_logger/appender/async.rb +86 -173
  5. data/lib/semantic_logger/appender/cloudwatch_logs.rb +4 -4
  6. data/lib/semantic_logger/appender/elasticsearch.rb +6 -182
  7. data/lib/semantic_logger/appender/elasticsearch_base.rb +212 -0
  8. data/lib/semantic_logger/appender/elasticsearch_http.rb +2 -2
  9. data/lib/semantic_logger/appender/file.rb +20 -4
  10. data/lib/semantic_logger/appender/graylog.rb +2 -2
  11. data/lib/semantic_logger/appender/honeybadger_insights.rb +1 -1
  12. data/lib/semantic_logger/appender/http.rb +27 -2
  13. data/lib/semantic_logger/appender/io.rb +8 -4
  14. data/lib/semantic_logger/appender/kafka.rb +2 -2
  15. data/lib/semantic_logger/appender/loki.rb +2 -4
  16. data/lib/semantic_logger/appender/mongodb.rb +3 -6
  17. data/lib/semantic_logger/appender/new_relic_logs.rb +12 -2
  18. data/lib/semantic_logger/appender/open_telemetry.rb +25 -11
  19. data/lib/semantic_logger/appender/opensearch.rb +35 -0
  20. data/lib/semantic_logger/appender/rabbitmq.rb +3 -3
  21. data/lib/semantic_logger/appender/sentry_ruby.rb +8 -3
  22. data/lib/semantic_logger/appender/splunk.rb +2 -2
  23. data/lib/semantic_logger/appender/splunk_http.rb +3 -3
  24. data/lib/semantic_logger/appender/syslog.rb +5 -4
  25. data/lib/semantic_logger/appender/tcp.rb +2 -2
  26. data/lib/semantic_logger/appender/udp.rb +2 -2
  27. data/lib/semantic_logger/appender/wrapper.rb +4 -4
  28. data/lib/semantic_logger/appender.rb +30 -19
  29. data/lib/semantic_logger/appenders.rb +26 -5
  30. data/lib/semantic_logger/base.rb +113 -21
  31. data/lib/semantic_logger/concerns/compatibility.rb +2 -2
  32. data/lib/semantic_logger/core_ext/process.rb +34 -0
  33. data/lib/semantic_logger/formatters/base.rb +46 -7
  34. data/lib/semantic_logger/formatters/color.rb +6 -3
  35. data/lib/semantic_logger/formatters/default.rb +6 -4
  36. data/lib/semantic_logger/formatters/ecs.rb +151 -0
  37. data/lib/semantic_logger/formatters/fluentd.rb +15 -4
  38. data/lib/semantic_logger/formatters/json.rb +6 -1
  39. data/lib/semantic_logger/formatters/logfmt.rb +2 -2
  40. data/lib/semantic_logger/formatters/loki.rb +4 -4
  41. data/lib/semantic_logger/formatters/new_relic_logs.rb +4 -6
  42. data/lib/semantic_logger/formatters/open_telemetry.rb +40 -2
  43. data/lib/semantic_logger/formatters/pattern.rb +235 -0
  44. data/lib/semantic_logger/formatters/raw.rb +2 -2
  45. data/lib/semantic_logger/formatters/signalfx.rb +2 -2
  46. data/lib/semantic_logger/formatters/syslog.rb +14 -3
  47. data/lib/semantic_logger/formatters/syslog_cee.rb +1 -1
  48. data/lib/semantic_logger/formatters.rb +2 -0
  49. data/lib/semantic_logger/log.rb +23 -4
  50. data/lib/semantic_logger/logger.rb +2 -2
  51. data/lib/semantic_logger/metric/new_relic.rb +2 -2
  52. data/lib/semantic_logger/metric/signalfx.rb +2 -2
  53. data/lib/semantic_logger/metric/statsd.rb +2 -2
  54. data/lib/semantic_logger/processor.rb +22 -1
  55. data/lib/semantic_logger/queue_processor.rb +369 -0
  56. data/lib/semantic_logger/reporters/minitest.rb +4 -4
  57. data/lib/semantic_logger/semantic_logger.rb +103 -11
  58. data/lib/semantic_logger/subscriber.rb +15 -2
  59. data/lib/semantic_logger/sync_processor.rb +25 -3
  60. data/lib/semantic_logger/test/capture_log_events.rb +2 -2
  61. data/lib/semantic_logger/test/minitest.rb +8 -4
  62. data/lib/semantic_logger/test/rspec.rb +249 -0
  63. data/lib/semantic_logger/utils.rb +83 -4
  64. data/lib/semantic_logger/version.rb +1 -1
  65. data/lib/semantic_logger.rb +9 -0
  66. metadata +15 -6
  67. data/lib/semantic_logger/appender/async_batch.rb +0 -93
@@ -34,6 +34,19 @@ module SemanticLogger
34
34
  @level || SemanticLogger.default_level
35
35
  end
36
36
 
37
+ # Set the logging level for this logger during the execution of the given block
38
+ #
39
+ # Refer to the documentation for `#level=` for more information about the possible log levels.
40
+ def with_level(new_level)
41
+ old_level = level
42
+
43
+ self.level = new_level
44
+
45
+ yield
46
+ ensure
47
+ self.level = old_level
48
+ end
49
+
37
50
  # Implement the log level calls
38
51
  # logger.debug(message, hash|exception=nil, &block)
39
52
  #
@@ -185,7 +198,18 @@ module SemanticLogger
185
198
  # to:
186
199
  # `logger.tagged('first', 'more', 'other')`
187
200
  # - For better performance with clean tags, see `SemanticLogger.tagged`.
201
+ #
202
+ # Child logger:
203
+ # - When called *without* a block, returns a new logger instance that permanently
204
+ # carries the supplied tags ("instance tags"). Those tags are added to every log
205
+ # entry emitted by the returned logger, and _only_ that logger, even across threads.
206
+ # Unlike the block form they are not pushed onto the thread, so other loggers are
207
+ # unaffected. This is the recommended way to bind a logger to an object's identity:
208
+ # logger = SemanticLogger['Cart'].tagged(cart_id: cart.id)
188
209
  def tagged(*tags)
210
+ # No block: build a child logger that carries these tags on every entry.
211
+ return tagged_child(tags) unless block_given?
212
+
189
213
  block = -> { yield(self) }
190
214
  # Allow named tags to be passed into the logger
191
215
  # Rails::Rack::Logger passes logs as an array with a single argument
@@ -213,6 +237,10 @@ module SemanticLogger
213
237
  SemanticLogger.named_tags
214
238
  end
215
239
 
240
+ # Tags permanently bound to this logger instance via `#tagged` (without a block).
241
+ # Empty for a regular (non-child) logger.
242
+ attr_reader :instance_tags, :instance_named_tags
243
+
216
244
  # Returns the list of tags pushed after flattening them out and removing blanks
217
245
  #
218
246
  # Note:
@@ -231,13 +259,13 @@ module SemanticLogger
231
259
  end
232
260
 
233
261
  # :nodoc:
234
- def silence(new_level = :error, &block)
235
- SemanticLogger.silence(new_level, &block)
262
+ def silence(new_level = :error, &)
263
+ SemanticLogger.silence(new_level, &)
236
264
  end
237
265
 
238
266
  # :nodoc:
239
- def fast_tag(tag, &block)
240
- SemanticLogger.fast_tag(tag, &block)
267
+ def fast_tag(tag, &)
268
+ SemanticLogger.fast_tag(tag, &)
241
269
  end
242
270
 
243
271
  # Write log data to underlying data storage
@@ -250,6 +278,11 @@ module SemanticLogger
250
278
  meets_log_level?(log) && !filtered?(log)
251
279
  end
252
280
 
281
+ protected
282
+
283
+ # Only assigned to when building a child logger via `#tagged` (see #tagged_child).
284
+ attr_writer :instance_tags, :instance_named_tags
285
+
253
286
  private
254
287
 
255
288
  # Initializer for Abstract Class SemanticLogger::Base
@@ -281,8 +314,13 @@ module SemanticLogger
281
314
  raise ":filter must be a Regexp, Proc, or implement :call"
282
315
  end
283
316
 
284
- @filter = filter.is_a?(Regexp) ? filter.freeze : filter
285
- @name = klass.is_a?(String) ? klass : klass.name
317
+ @filter = filter.is_a?(Regexp) ? filter.freeze : filter
318
+ @name = klass.is_a?(String) ? klass : klass.name
319
+ # `[].freeze` / `{}.freeze` compile to `opt_ary_freeze` / `opt_hash_freeze`, which return
320
+ # a shared frozen singleton. Every logger points at the same two objects, so these default
321
+ # (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
322
+ @instance_tags = [].freeze
323
+ @instance_named_tags = {}.freeze
286
324
  if level.nil?
287
325
  # Allow the global default level to determine this loggers log level
288
326
  @level_index = nil
@@ -311,13 +349,59 @@ module SemanticLogger
311
349
  (level_index <= (log.level_index || 0))
312
350
  end
313
351
 
352
+ # Build a new Log entry, layering this logger's instance tags on top of the
353
+ # current thread context.
354
+ def new_log(level, index = nil)
355
+ apply_instance_tags(Log.new(name, level, index))
356
+ end
357
+
358
+ # Merge this logger's instance tags into the supplied Log entry.
359
+ #
360
+ # Composition (mirrors the per-logger merge rule proposed in PR #301):
361
+ # - Positional: thread tags first, then this logger's instance tags appended.
362
+ # - Named: instance named tags merged on top of the thread named tags, so on a
363
+ # key conflict the logger's own identity wins (it is the more specific value).
364
+ # Only this logger's entries are affected; the thread context is left untouched.
365
+ def apply_instance_tags(log)
366
+ unless @instance_tags.empty?
367
+ log.tags = log.tags.empty? ? @instance_tags.dup : log.tags + @instance_tags
368
+ end
369
+ unless @instance_named_tags.empty?
370
+ log.named_tags = log.named_tags.empty? ? @instance_named_tags.dup : log.named_tags.merge(@instance_named_tags)
371
+ end
372
+ log
373
+ end
374
+
375
+ # Returns a copy of this logger that permanently carries the supplied tags.
376
+ # Positional (string) and named (Hash) tags may be mixed; they are merged on
377
+ # top of any instance tags this logger already carries.
378
+ def tagged_child(tags)
379
+ positional = @instance_tags.dup
380
+ named = @instance_named_tags.dup
381
+ tags.flatten.each do |tag|
382
+ case tag
383
+ when Hash
384
+ named = named.merge(tag)
385
+ when nil, ""
386
+ # Ignore blanks for Rails compatibility, matching the block form.
387
+ else
388
+ positional << tag.to_s
389
+ end
390
+ end
391
+
392
+ child = clone
393
+ child.instance_tags = positional.freeze
394
+ child.instance_named_tags = named.freeze
395
+ child
396
+ end
397
+
314
398
  # Log message at the specified level
315
- def log_internal(level, index, message = nil, payload = nil, exception = nil)
399
+ def log_internal(level, index, message = nil, payload = nil, exception = nil, &block)
316
400
  # Handle variable number of arguments by detecting exception object and payload hash.
317
401
  if exception.nil? && payload.nil? && message.respond_to?(:backtrace) && message.respond_to?(:message)
318
402
  exception = message
319
403
  message = nil
320
- elsif exception.nil? && payload && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
404
+ elsif exception.nil? && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
321
405
  exception = payload
322
406
  payload = nil
323
407
  elsif payload && !payload.is_a?(Hash)
@@ -325,12 +409,12 @@ module SemanticLogger
325
409
  payload = nil
326
410
  end
327
411
 
328
- log = Log.new(name, level, index)
412
+ log = new_log(level, index)
329
413
  should_log =
330
414
  if exception.nil? && payload.nil? && message.is_a?(Hash)
331
415
  # All arguments as a hash in the message.
332
416
  log.assign(**log.extract_arguments(message))
333
- elsif exception.nil? && message && payload && payload.is_a?(Hash)
417
+ elsif exception.nil? && message && payload.is_a?(Hash)
334
418
  # Message supplied along with a hash with the remaining arguments.
335
419
  log.assign(**log.extract_arguments(payload, message))
336
420
  else
@@ -339,20 +423,28 @@ module SemanticLogger
339
423
  end
340
424
 
341
425
  # Add result of block to message or payload if not nil
342
- if block_given?
343
- result = yield(log)
344
- case result
345
- when String
346
- log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
347
- when Hash
348
- log.assign_hash(result)
349
- end
350
- end
426
+ apply_block_result(log, &block) if block
351
427
 
352
428
  # Log level may change during assign due to :on_exception_level
353
429
  self.log(log) if should_log && should_log?(log)
354
430
  end
355
431
 
432
+ # Merge the result of a logging block into the log entry.
433
+ #
434
+ # Zero-arity blocks (e.g. `logger.info { "msg" }` or a zero-arity lambda) are called without
435
+ # arguments. Lambdas enforce their arity, so yielding the log to a zero-arity lambda would
436
+ # raise ArgumentError. Blocks that accept an argument still receive the log so they can read
437
+ # or mutate it.
438
+ def apply_block_result(log, &block)
439
+ result = block.arity.zero? ? block.call : block.call(log)
440
+ case result
441
+ when String
442
+ log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
443
+ when Hash
444
+ log.assign_hash(result)
445
+ end
446
+ end
447
+
356
448
  # Measure the supplied block and log the message
357
449
  def measure_internal(level, index, message, params)
358
450
  exception = nil
@@ -376,7 +468,7 @@ module SemanticLogger
376
468
  exception = e
377
469
  ensure
378
470
  # Must use ensure block otherwise a `return` in the yield above will skip the log entry
379
- log = Log.new(name, level, index)
471
+ log = new_log(level, index)
380
472
  exception ||= params[:exception]
381
473
  message = params[:message] if params[:message]
382
474
  duration =
@@ -424,7 +516,7 @@ module SemanticLogger
424
516
  rescue Exception => e
425
517
  exception = e
426
518
  ensure
427
- log = Log.new(name, level, index)
519
+ log = new_log(level, index)
428
520
  # May return false due to elastic logging
429
521
  should_log = log.assign(
430
522
  message: message,
@@ -37,11 +37,11 @@ module SemanticLogger
37
37
  end
38
38
 
39
39
  # :nodoc:
40
- def add(severity, message = nil, progname = nil, &block)
40
+ def add(severity, message = nil, progname = nil, &)
41
41
  index = Levels.index(severity)
42
42
  if level_index <= index
43
43
  level = Levels.level(index)
44
- log_internal(level, index, message, progname, &block)
44
+ log_internal(level, index, message, progname, &)
45
45
  true
46
46
  else
47
47
  false
@@ -0,0 +1,34 @@
1
+ module SemanticLogger
2
+ module CoreExt
3
+ # Reopen all appenders in the child process after a fork.
4
+ #
5
+ # Prepended onto `Process.singleton_class` when Semantic Logger is loaded.
6
+ # Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
7
+ #
8
+ # `Process._fork` (Ruby 3.1+) is the single method that every fork path routes
9
+ # through (`Kernel#fork`, `Process.fork`, `IO.popen`, `Kernel#system`, and
10
+ # backticks), so overriding it covers them all with one hook.
11
+ #
12
+ # Note: both `_fork` and `daemon` must be overridden to reliably restart the
13
+ # appender thread, since `Process.daemon` does not route through `_fork`
14
+ # (https://bugs.ruby-lang.org/issues/18911). Do not collapse this down to only
15
+ # `_fork`.
16
+ module Process
17
+ # `_fork` runs in both the parent and the child. It returns 0 in the child
18
+ # and the child's pid in the parent, so only reopen in the child. Reopening
19
+ # in the parent would needlessly recreate its queue and worker thread,
20
+ # risking the loss of messages still on the queue.
21
+ def _fork
22
+ child_pid = super
23
+ SemanticLogger.reopen if child_pid.zero? && SemanticLogger.reopen_on_fork?
24
+ child_pid
25
+ end
26
+
27
+ # `Process.daemon` does not route through `_fork`, so reopen explicitly. Once
28
+ # it returns, the caller is running as the daemon (child) process.
29
+ def daemon(...)
30
+ super.tap { SemanticLogger.reopen if SemanticLogger.reopen_on_fork? }
31
+ end
32
+ end
33
+ end
34
+ end
@@ -2,7 +2,21 @@ require "time"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Base
5
- attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision
5
+ attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
6
+ :escape_control_chars
7
+
8
+ # Printable escapes for the most common control characters. Any other
9
+ # control character is escaped to its hexadecimal `\xHH` form by
10
+ # #escape_control_characters.
11
+ CONTROL_CHAR_ESCAPES = {
12
+ "\n" => "\\n",
13
+ "\r" => "\\r",
14
+ "\t" => "\\t",
15
+ "\e" => "\\e"
16
+ }.freeze
17
+
18
+ # Matches C0 control characters (including newlines and the ANSI escape) and DEL.
19
+ CONTROL_CHARS = /[\x00-\x1f\x7f]/
6
20
 
7
21
  # Time precision varies by Ruby interpreter
8
22
  # JRuby 9.1.8.0 supports microseconds
@@ -34,16 +48,27 @@ module SemanticLogger
34
48
  # precision: [Integer]
35
49
  # How many fractional digits to log times with.
36
50
  # Default: PRECISION (6, except on older JRuby, where 3)
51
+ # escape_control_chars: [Boolean]
52
+ # Replace control characters (newlines, the ANSI escape, etc.) in
53
+ # untrusted log data (message, tags, named tags, and exception
54
+ # message) with a printable escaped form, e.g. "\n".
55
+ # This prevents log forging and terminal escape-sequence injection
56
+ # when logging data that may contain attacker-controlled content.
57
+ # Note: Has no effect on structured formatters such as :json, which
58
+ # already escape control characters via JSON encoding.
59
+ # Default: false (preserve newlines and ANSI colors in text output)
37
60
  def initialize(time_format: nil,
38
61
  log_host: true,
39
62
  log_application: true,
40
63
  log_environment: true,
41
- precision: PRECISION)
42
- @time_format = time_format || self.class.build_time_format(precision)
43
- @log_host = log_host
44
- @log_application = log_application
45
- @log_environment = log_environment
46
- @precision = precision
64
+ precision: PRECISION,
65
+ escape_control_chars: false)
66
+ @time_format = time_format || self.class.build_time_format(precision)
67
+ @log_host = log_host
68
+ @log_application = log_application
69
+ @log_environment = log_environment
70
+ @precision = precision
71
+ @escape_control_chars = escape_control_chars
47
72
  end
48
73
 
49
74
  # Return default time format string
@@ -68,6 +93,20 @@ module SemanticLogger
68
93
 
69
94
  private
70
95
 
96
+ # When `escape_control_chars` is enabled, return a copy of the supplied
97
+ # value with any control characters replaced by a printable escaped form
98
+ # so that untrusted log data cannot forge log entries or inject terminal
99
+ # escape sequences. Otherwise the value is returned unchanged.
100
+ #
101
+ # Note: This escapes (preserves) control characters, and is opt-in. It is
102
+ # distinct from Log#cleansed_message, which unconditionally *strips* ANSI
103
+ # colorization from the message for structured (JSON/Loki) output.
104
+ def escape_control_characters(value)
105
+ return value unless escape_control_chars && value
106
+
107
+ value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
108
+ end
109
+
71
110
  # Return the Time as a formatted string
72
111
  def format_time(time)
73
112
  time = time.dup
@@ -85,7 +85,10 @@ module SemanticLogger
85
85
  end
86
86
 
87
87
  def tags
88
- "[#{color}#{log.tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]" if log.tags && !log.tags.empty?
88
+ return if log.tags.nil? || log.tags.empty?
89
+
90
+ tags = log.tags.map { |tag| escape_control_characters(tag) }
91
+ "[#{color}#{tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]"
89
92
  end
90
93
 
91
94
  # Named Tags
@@ -94,7 +97,7 @@ module SemanticLogger
94
97
  return if named_tags.nil? || named_tags.empty?
95
98
 
96
99
  list = []
97
- named_tags.each_pair { |name, value| list << "#{color}#{name}: #{value}#{color_map.clear}" }
100
+ named_tags.each_pair { |name, value| list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}" }
98
101
  "{#{list.join(', ')}}"
99
102
  end
100
103
 
@@ -123,7 +126,7 @@ module SemanticLogger
123
126
  def exception
124
127
  return unless log.exception
125
128
 
126
- "-- Exception: #{color}#{log.exception.class}: #{log.exception.message}#{color_map.clear}\n#{log.backtrace_to_s}"
129
+ "-- Exception: #{color}#{log.exception.class}: #{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
127
130
  end
128
131
 
129
132
  def call(log, logger)
@@ -32,7 +32,9 @@ module SemanticLogger
32
32
 
33
33
  # Tags
34
34
  def tags
35
- "[#{log.tags.join('] [')}]" if log.tags && !log.tags.empty?
35
+ return if log.tags.nil? || log.tags.empty?
36
+
37
+ "[#{log.tags.map { |tag| escape_control_characters(tag) }.join('] [')}]"
36
38
  end
37
39
 
38
40
  # Named Tags
@@ -41,7 +43,7 @@ module SemanticLogger
41
43
  return if named_tags.nil? || named_tags.empty?
42
44
 
43
45
  list = []
44
- named_tags.each_pair { |name, value| list << "#{name}: #{value}" }
46
+ named_tags.each_pair { |name, value| list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}" }
45
47
  "{#{list.join(', ')}}"
46
48
  end
47
49
 
@@ -57,7 +59,7 @@ module SemanticLogger
57
59
 
58
60
  # Log message
59
61
  def message
60
- "-- #{log.message}" if log.message
62
+ "-- #{escape_control_characters(log.message)}" if log.message
61
63
  end
62
64
 
63
65
  # Payload
@@ -70,7 +72,7 @@ module SemanticLogger
70
72
 
71
73
  # Exception
72
74
  def exception
73
- "-- Exception: #{log.exception.class}: #{log.exception.message}\n#{log.backtrace_to_s}" if log.exception
75
+ "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n#{log.backtrace_to_s}" if log.exception
74
76
  end
75
77
 
76
78
  # Default text log format
@@ -0,0 +1,151 @@
1
+ require "json"
2
+ module SemanticLogger
3
+ module Formatters
4
+ # Formatter conforming to the Elastic Common Schema (ECS).
5
+ #
6
+ # Emits log events using the nested field names defined by ECS so that they
7
+ # integrate cleanly with Filebeat and the Elastic stack (Elasticsearch,
8
+ # Kibana) without requiring an ingest pipeline to rename fields.
9
+ #
10
+ # Usage:
11
+ # SemanticLogger.add_appender(io: $stdout, formatter: :ecs)
12
+ #
13
+ # # Route the payload, metric, and other SemanticLogger-specific data into
14
+ # # a custom top-level namespace (default "semantic_logger"):
15
+ # SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: "my_app"}})
16
+ #
17
+ # # Or merge the payload directly into ECS `labels` instead of a namespace:
18
+ # SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: nil}})
19
+ #
20
+ # == Field mapping (SemanticLogger -> ECS 8.x)
21
+ # time -> @timestamp (ISO-8601)
22
+ # level -> log.level
23
+ # name -> log.logger
24
+ # file_name / line -> log.origin.file.name / log.origin.file.line
25
+ # message -> message
26
+ # thread_name -> process.thread.name
27
+ # pid -> process.pid
28
+ # host -> host.hostname
29
+ # application -> service.name
30
+ # environment -> service.environment
31
+ # exception -> error.type / error.message / error.stack_trace
32
+ # duration -> event.duration (nanoseconds, as required by ECS)
33
+ # tags -> tags (ECS top-level array)
34
+ # named_tags -> labels.* (scalar key/value pairs)
35
+ # payload -> <namespace>.* (or labels.* when namespace is nil)
36
+ # metric/metric_amount -> <namespace>.metric / <namespace>.metric_amount
37
+ #
38
+ # == Reference
39
+ # * https://www.elastic.co/docs/reference/ecs
40
+ # * https://www.elastic.co/docs/reference/ecs/ecs-custom-fields-in-ecs
41
+ class Ecs < Raw
42
+ # ECS version this formatter targets.
43
+ ECS_VERSION = "8.11.0".freeze
44
+
45
+ # namespace: [String|Symbol|nil]
46
+ # Top-level field set used to hold SemanticLogger-specific data that has
47
+ # no native ECS home (payload, metric, metric_amount). A proper-noun
48
+ # namespace is guaranteed never to collide with a current or future ECS
49
+ # field. Set to nil to merge the payload into ECS `labels` instead.
50
+ # Default: "semantic_logger"
51
+ attr_reader :namespace
52
+
53
+ def initialize(namespace: "semantic_logger", time_format: :iso_8601, time_key: :timestamp, **args)
54
+ @namespace = namespace&.to_sym
55
+
56
+ super(time_format: time_format, time_key: time_key, **args)
57
+ end
58
+
59
+ # Returns the log event as a single line of ECS-formatted JSON, so it can
60
+ # be written to stdout / a file and shipped by Filebeat or Elastic Agent.
61
+ def call(log, logger)
62
+ Utils.to_json(ecs_hash(super))
63
+ end
64
+
65
+ # Returns a batch of log events as a single JSON array.
66
+ def batch(logs, logger)
67
+ "[#{logs.map { |log| call(log, logger) }.join(',')}]"
68
+ end
69
+
70
+ private
71
+
72
+ # Remap the flat hash built by Raw#call into the nested ECS field layout.
73
+ def ecs_hash(hash)
74
+ result = base(hash)
75
+ result[:process] = process(hash)
76
+ result[:host] = {hostname: hash[:host]} if hash[:host]
77
+ add_service(result, hash)
78
+ add_origin(result, hash)
79
+ add_event(result, hash)
80
+ result[:tags] = hash[:tags] if hash[:tags]
81
+ add_error(result, hash)
82
+ add_extras(result, hash)
83
+ result
84
+ end
85
+
86
+ def base(hash)
87
+ log = {level: hash[:level].to_s}
88
+ log[:logger] = hash[:name] if hash[:name]
89
+ {
90
+ "@timestamp": hash[:timestamp],
91
+ message: hash[:message],
92
+ ecs: {version: ECS_VERSION},
93
+ log: log
94
+ }
95
+ end
96
+
97
+ def process(hash)
98
+ result = {pid: hash[:pid]}
99
+ result[:thread] = {name: hash[:thread].to_s} if hash[:thread]
100
+ result
101
+ end
102
+
103
+ def add_service(result, hash)
104
+ service = {}
105
+ service[:name] = hash[:application] if hash[:application]
106
+ service[:environment] = hash[:environment] if hash[:environment]
107
+ result[:service] = service unless service.empty?
108
+ end
109
+
110
+ def add_origin(result, hash)
111
+ return unless hash[:file]
112
+
113
+ result[:log][:origin] = {file: {name: hash[:file], line: hash[:line]}.compact}
114
+ end
115
+
116
+ # ECS event.duration is measured in nanoseconds.
117
+ def add_event(result, hash)
118
+ return unless hash[:duration_ms]
119
+
120
+ result[:event] = {duration: (hash[:duration_ms] * 1_000_000).round}
121
+ end
122
+
123
+ def add_error(result, hash)
124
+ return unless hash[:exception]
125
+
126
+ result[:error] = {
127
+ type: hash[:exception][:name],
128
+ message: hash[:exception][:message],
129
+ stack_trace: Array(hash[:exception][:stack_trace]).join("\n")
130
+ }
131
+ end
132
+
133
+ # Place SemanticLogger-specific data (payload, metric) that has no native
134
+ # ECS home into the configured namespace, plus named_tags into labels.
135
+ def add_extras(result, hash)
136
+ labels = hash[:named_tags].is_a?(Hash) ? hash[:named_tags].dup : {}
137
+
138
+ extra = {}
139
+ extra[:payload] = hash[:payload] if hash[:payload]
140
+ extra[:metric] = hash[:metric] if hash[:metric]
141
+ extra[:metric_amount] = hash[:metric_amount] if hash[:metric_amount]
142
+
143
+ unless extra.empty?
144
+ namespace ? result[namespace] = extra : labels.merge!(extra)
145
+ end
146
+
147
+ result[:labels] = labels unless labels.empty?
148
+ end
149
+ end
150
+ end
151
+ end
@@ -7,9 +7,9 @@ module SemanticLogger
7
7
  class Fluentd < Json
8
8
  attr_reader :need_process_info
9
9
 
10
- def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
10
+ def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, log_host: false, **args)
11
11
  @need_process_info = need_process_info
12
- super(time_format: time_format, time_key: time_key, **args)
12
+ super(time_format: time_format, time_key: time_key, log_host: log_host, **args)
13
13
  end
14
14
 
15
15
  def level
@@ -17,8 +17,19 @@ module SemanticLogger
17
17
  hash["severity_index"] = log.level_index
18
18
  end
19
19
 
20
- def process_info
21
- # Ignore fields: pid, thread, file and line by default
20
+ # Ignore process fields: pid, thread, file and line by default.
21
+ # These are rarely useful under Fluentd (e.g. containerized processes
22
+ # usually have pid 1), so they are only included when explicitly requested
23
+ # via `need_process_info: true`.
24
+ def pid
25
+ super if need_process_info
26
+ end
27
+
28
+ def thread_name
29
+ super if need_process_info
30
+ end
31
+
32
+ def file_name_and_line
22
33
  super if need_process_info
23
34
  end
24
35
  end
@@ -9,7 +9,12 @@ module SemanticLogger
9
9
 
10
10
  # Returns log messages in JSON format
11
11
  def call(log, logger)
12
- super.to_json
12
+ Utils.to_json(super)
13
+ end
14
+
15
+ # Returns a batch of log messages as a single JSON array.
16
+ def batch(logs, logger)
17
+ "[#{logs.map { |log| call(log, logger) }.join(',')}]"
13
18
  end
14
19
  end
15
20
  end
@@ -64,9 +64,9 @@ module SemanticLogger
64
64
  flattened = @parsed.map do |key, value|
65
65
  case value
66
66
  when Hash, Array
67
- "#{key}=#{value.to_s.to_json}"
67
+ "#{key}=#{Utils.to_json(value.to_s)}"
68
68
  else
69
- "#{key}=#{value.to_json}"
69
+ "#{key}=#{Utils.to_json(value)}"
70
70
  end
71
71
  end
72
72
 
@@ -10,7 +10,7 @@ module SemanticLogger
10
10
  self.logger = logger
11
11
  self.log = log
12
12
 
13
- {streams: [build_stream]}.to_json
13
+ Utils.to_json({streams: [build_stream]})
14
14
  end
15
15
 
16
16
  # Returns [String] a JSON batch of logs
@@ -22,7 +22,7 @@ module SemanticLogger
22
22
  build_stream
23
23
  end
24
24
 
25
- {streams: streams}.to_json
25
+ Utils.to_json({streams: streams})
26
26
  end
27
27
 
28
28
  private
@@ -82,7 +82,7 @@ module SemanticLogger
82
82
 
83
83
  log.context.each do |key, value|
84
84
  serialized_value = if value.is_a?(Hash)
85
- value.to_json
85
+ Utils.to_json(value)
86
86
  else
87
87
  value.to_s
88
88
  end
@@ -144,7 +144,7 @@ module SemanticLogger
144
144
 
145
145
  result[string_key] = case value
146
146
  when Hash
147
- JSON.generate(stringify_hash(value))
147
+ Utils.to_json(stringify_hash(value))
148
148
  else
149
149
  value.to_s
150
150
  end