semantic_logger 4.18.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +51 -81
  3. data/Rakefile +39 -3
  4. data/docs/api.md +480 -0
  5. data/docs/appenders.md +990 -0
  6. data/docs/config.md +605 -0
  7. data/docs/index.md +197 -0
  8. data/docs/log.md +108 -0
  9. data/docs/metrics.md +168 -0
  10. data/docs/operations.md +400 -0
  11. data/docs/rails.md +816 -0
  12. data/docs/security.md +119 -0
  13. data/docs/testing.md +285 -0
  14. data/docs/upgrading.md +409 -0
  15. data/lib/semantic_logger/appender/async.rb +86 -173
  16. data/lib/semantic_logger/appender/cloudwatch_logs.rb +8 -6
  17. data/lib/semantic_logger/appender/elasticsearch.rb +8 -187
  18. data/lib/semantic_logger/appender/elasticsearch_base.rb +212 -0
  19. data/lib/semantic_logger/appender/elasticsearch_http.rb +16 -9
  20. data/lib/semantic_logger/appender/file.rb +25 -4
  21. data/lib/semantic_logger/appender/graylog.rb +2 -2
  22. data/lib/semantic_logger/appender/http.rb +27 -2
  23. data/lib/semantic_logger/appender/io.rb +8 -4
  24. data/lib/semantic_logger/appender/kafka.rb +2 -2
  25. data/lib/semantic_logger/appender/loki.rb +2 -4
  26. data/lib/semantic_logger/appender/mongodb.rb +3 -6
  27. data/lib/semantic_logger/appender/new_relic.rb +6 -0
  28. data/lib/semantic_logger/appender/new_relic_logs.rb +4 -3
  29. data/lib/semantic_logger/appender/open_telemetry.rb +10 -7
  30. data/lib/semantic_logger/appender/opensearch.rb +36 -0
  31. data/lib/semantic_logger/appender/rabbitmq.rb +5 -4
  32. data/lib/semantic_logger/appender/sentry.rb +5 -0
  33. data/lib/semantic_logger/appender/sentry_ruby.rb +2 -2
  34. data/lib/semantic_logger/appender/splunk.rb +2 -2
  35. data/lib/semantic_logger/appender/splunk_http.rb +3 -3
  36. data/lib/semantic_logger/appender/syslog.rb +11 -7
  37. data/lib/semantic_logger/appender/tcp.rb +4 -3
  38. data/lib/semantic_logger/appender/udp.rb +2 -2
  39. data/lib/semantic_logger/appender/wrapper.rb +5 -4
  40. data/lib/semantic_logger/appender.rb +36 -22
  41. data/lib/semantic_logger/appenders.rb +27 -5
  42. data/lib/semantic_logger/base.rb +100 -23
  43. data/lib/semantic_logger/concerns/compatibility.rb +2 -2
  44. data/lib/semantic_logger/core_ext/process.rb +34 -0
  45. data/lib/semantic_logger/formatters/base.rb +46 -7
  46. data/lib/semantic_logger/formatters/color.rb +9 -3
  47. data/lib/semantic_logger/formatters/default.rb +11 -4
  48. data/lib/semantic_logger/formatters/ecs.rb +151 -0
  49. data/lib/semantic_logger/formatters/fluentd.rb +15 -4
  50. data/lib/semantic_logger/formatters/json.rb +6 -1
  51. data/lib/semantic_logger/formatters/logfmt.rb +6 -4
  52. data/lib/semantic_logger/formatters/loki.rb +4 -4
  53. data/lib/semantic_logger/formatters/open_telemetry.rb +15 -5
  54. data/lib/semantic_logger/formatters/pattern.rb +237 -0
  55. data/lib/semantic_logger/formatters/raw.rb +2 -2
  56. data/lib/semantic_logger/formatters/signalfx.rb +2 -2
  57. data/lib/semantic_logger/formatters/syslog.rb +16 -4
  58. data/lib/semantic_logger/formatters/syslog_cee.rb +3 -2
  59. data/lib/semantic_logger/formatters.rb +2 -0
  60. data/lib/semantic_logger/log.rb +18 -4
  61. data/lib/semantic_logger/logger.rb +2 -2
  62. data/lib/semantic_logger/metric/new_relic.rb +2 -2
  63. data/lib/semantic_logger/metric/signalfx.rb +6 -4
  64. data/lib/semantic_logger/metric/statsd.rb +2 -2
  65. data/lib/semantic_logger/processor.rb +21 -0
  66. data/lib/semantic_logger/queue_processor.rb +372 -0
  67. data/lib/semantic_logger/reporters/minitest.rb +8 -8
  68. data/lib/semantic_logger/semantic_logger.rb +103 -11
  69. data/lib/semantic_logger/subscriber.rb +15 -2
  70. data/lib/semantic_logger/sync_processor.rb +25 -3
  71. data/lib/semantic_logger/test/capture_log_events.rb +3 -3
  72. data/lib/semantic_logger/test/minitest.rb +8 -4
  73. data/lib/semantic_logger/test/rspec.rb +249 -0
  74. data/lib/semantic_logger/utils.rb +83 -4
  75. data/lib/semantic_logger/version.rb +1 -1
  76. data/lib/semantic_logger.rb +9 -0
  77. metadata +28 -8
  78. data/lib/semantic_logger/appender/async_batch.rb +0 -93
@@ -155,8 +155,6 @@ module SemanticLogger
155
155
  log.named_tags = (thread[:semantic_logger_named_tags] || {}).clone
156
156
  thread.backtrace
157
157
  end
158
- # TODO: Keep backtrace instead of transforming into a text message at this point
159
- # Maybe log_backtrace: true
160
158
  if backtrace
161
159
  message += "\n"
162
160
  message << backtrace.join("\n")
@@ -198,7 +196,18 @@ module SemanticLogger
198
196
  # to:
199
197
  # `logger.tagged('first', 'more', 'other')`
200
198
  # - For better performance with clean tags, see `SemanticLogger.tagged`.
199
+ #
200
+ # Child logger:
201
+ # - When called *without* a block, returns a new logger instance that permanently
202
+ # carries the supplied tags ("instance tags"). Those tags are added to every log
203
+ # entry emitted by the returned logger, and _only_ that logger, even across threads.
204
+ # Unlike the block form they are not pushed onto the thread, so other loggers are
205
+ # unaffected. This is the recommended way to bind a logger to an object's identity:
206
+ # logger = SemanticLogger['Cart'].tagged(cart_id: cart.id)
201
207
  def tagged(*tags)
208
+ # No block: build a child logger that carries these tags on every entry.
209
+ return tagged_child(tags) unless block_given?
210
+
202
211
  block = -> { yield(self) }
203
212
  # Allow named tags to be passed into the logger
204
213
  # Rails::Rack::Logger passes logs as an array with a single argument
@@ -226,6 +235,10 @@ module SemanticLogger
226
235
  SemanticLogger.named_tags
227
236
  end
228
237
 
238
+ # Tags permanently bound to this logger instance via `#tagged` (without a block).
239
+ # Empty for a regular (non-child) logger.
240
+ attr_reader :instance_tags, :instance_named_tags
241
+
229
242
  # Returns the list of tags pushed after flattening them out and removing blanks
230
243
  #
231
244
  # Note:
@@ -244,13 +257,13 @@ module SemanticLogger
244
257
  end
245
258
 
246
259
  # :nodoc:
247
- def silence(new_level = :error, &block)
248
- SemanticLogger.silence(new_level, &block)
260
+ def silence(new_level = :error, &)
261
+ SemanticLogger.silence(new_level, &)
249
262
  end
250
263
 
251
264
  # :nodoc:
252
- def fast_tag(tag, &block)
253
- SemanticLogger.fast_tag(tag, &block)
265
+ def fast_tag(tag, &)
266
+ SemanticLogger.fast_tag(tag, &)
254
267
  end
255
268
 
256
269
  # Write log data to underlying data storage
@@ -263,6 +276,11 @@ module SemanticLogger
263
276
  meets_log_level?(log) && !filtered?(log)
264
277
  end
265
278
 
279
+ protected
280
+
281
+ # Only assigned to when building a child logger via `#tagged` (see #tagged_child).
282
+ attr_writer :instance_tags, :instance_named_tags
283
+
266
284
  private
267
285
 
268
286
  # Initializer for Abstract Class SemanticLogger::Base
@@ -294,8 +312,13 @@ module SemanticLogger
294
312
  raise ":filter must be a Regexp, Proc, or implement :call"
295
313
  end
296
314
 
297
- @filter = filter.is_a?(Regexp) ? filter.freeze : filter
298
- @name = klass.is_a?(String) ? klass : klass.name
315
+ @filter = filter.is_a?(Regexp) ? filter.freeze : filter
316
+ @name = klass.is_a?(String) ? klass : klass.name
317
+ # `[].freeze` / `{}.freeze` compile to `opt_ary_freeze` / `opt_hash_freeze`, which return
318
+ # a shared frozen singleton. Every logger points at the same two objects, so these default
319
+ # (empty) instance tags allocate no garbage. Child loggers replace them via #tagged_child.
320
+ @instance_tags = [].freeze
321
+ @instance_named_tags = {}.freeze
299
322
  if level.nil?
300
323
  # Allow the global default level to determine this loggers log level
301
324
  @level_index = nil
@@ -324,13 +347,59 @@ module SemanticLogger
324
347
  (level_index <= (log.level_index || 0))
325
348
  end
326
349
 
350
+ # Build a new Log entry, layering this logger's instance tags on top of the
351
+ # current thread context.
352
+ def new_log(level, index = nil)
353
+ apply_instance_tags(Log.new(name, level, index))
354
+ end
355
+
356
+ # Merge this logger's instance tags into the supplied Log entry.
357
+ #
358
+ # Composition (mirrors the per-logger merge rule proposed in PR #301):
359
+ # - Positional: thread tags first, then this logger's instance tags appended.
360
+ # - Named: instance named tags merged on top of the thread named tags, so on a
361
+ # key conflict the logger's own identity wins (it is the more specific value).
362
+ # Only this logger's entries are affected; the thread context is left untouched.
363
+ def apply_instance_tags(log)
364
+ unless @instance_tags.empty?
365
+ log.tags = log.tags.empty? ? @instance_tags.dup : log.tags + @instance_tags
366
+ end
367
+ unless @instance_named_tags.empty?
368
+ log.named_tags = log.named_tags.empty? ? @instance_named_tags.dup : log.named_tags.merge(@instance_named_tags)
369
+ end
370
+ log
371
+ end
372
+
373
+ # Returns a copy of this logger that permanently carries the supplied tags.
374
+ # Positional (string) and named (Hash) tags may be mixed; they are merged on
375
+ # top of any instance tags this logger already carries.
376
+ def tagged_child(tags)
377
+ positional = @instance_tags.dup
378
+ named = @instance_named_tags.dup
379
+ tags.flatten.each do |tag|
380
+ case tag
381
+ when Hash
382
+ named = named.merge(tag)
383
+ when nil, ""
384
+ # Ignore blanks for Rails compatibility, matching the block form.
385
+ else
386
+ positional << tag.to_s
387
+ end
388
+ end
389
+
390
+ child = clone
391
+ child.instance_tags = positional.freeze
392
+ child.instance_named_tags = named.freeze
393
+ child
394
+ end
395
+
327
396
  # Log message at the specified level
328
- def log_internal(level, index, message = nil, payload = nil, exception = nil)
397
+ def log_internal(level, index, message = nil, payload = nil, exception = nil, &block)
329
398
  # Handle variable number of arguments by detecting exception object and payload hash.
330
399
  if exception.nil? && payload.nil? && message.respond_to?(:backtrace) && message.respond_to?(:message)
331
400
  exception = message
332
401
  message = nil
333
- elsif exception.nil? && payload && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
402
+ elsif exception.nil? && payload.respond_to?(:backtrace) && payload.respond_to?(:message)
334
403
  exception = payload
335
404
  payload = nil
336
405
  elsif payload && !payload.is_a?(Hash)
@@ -338,12 +407,12 @@ module SemanticLogger
338
407
  payload = nil
339
408
  end
340
409
 
341
- log = Log.new(name, level, index)
410
+ log = new_log(level, index)
342
411
  should_log =
343
412
  if exception.nil? && payload.nil? && message.is_a?(Hash)
344
413
  # All arguments as a hash in the message.
345
414
  log.assign(**log.extract_arguments(message))
346
- elsif exception.nil? && message && payload && payload.is_a?(Hash)
415
+ elsif exception.nil? && message && payload.is_a?(Hash)
347
416
  # Message supplied along with a hash with the remaining arguments.
348
417
  log.assign(**log.extract_arguments(payload, message))
349
418
  else
@@ -352,20 +421,28 @@ module SemanticLogger
352
421
  end
353
422
 
354
423
  # Add result of block to message or payload if not nil
355
- if block_given?
356
- result = yield(log)
357
- case result
358
- when String
359
- log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
360
- when Hash
361
- log.assign_hash(result)
362
- end
363
- end
424
+ apply_block_result(log, &block) if block
364
425
 
365
426
  # Log level may change during assign due to :on_exception_level
366
427
  self.log(log) if should_log && should_log?(log)
367
428
  end
368
429
 
430
+ # Merge the result of a logging block into the log entry.
431
+ #
432
+ # Zero-arity blocks (e.g. `logger.info { "msg" }` or a zero-arity lambda) are called without
433
+ # arguments. Lambdas enforce their arity, so yielding the log to a zero-arity lambda would
434
+ # raise ArgumentError. Blocks that accept an argument still receive the log so they can read
435
+ # or mutate it.
436
+ def apply_block_result(log, &block)
437
+ result = block.arity.zero? ? block.call : block.call(log)
438
+ case result
439
+ when String
440
+ log.message = log.message.nil? ? result : "#{log.message} -- #{result}"
441
+ when Hash
442
+ log.assign_hash(result)
443
+ end
444
+ end
445
+
369
446
  # Measure the supplied block and log the message
370
447
  def measure_internal(level, index, message, params)
371
448
  exception = nil
@@ -389,7 +466,7 @@ module SemanticLogger
389
466
  exception = e
390
467
  ensure
391
468
  # Must use ensure block otherwise a `return` in the yield above will skip the log entry
392
- log = Log.new(name, level, index)
469
+ log = new_log(level, index)
393
470
  exception ||= params[:exception]
394
471
  message = params[:message] if params[:message]
395
472
  duration =
@@ -437,7 +514,7 @@ module SemanticLogger
437
514
  rescue Exception => e
438
515
  exception = e
439
516
  ensure
440
- log = Log.new(name, level, index)
517
+ log = new_log(level, index)
441
518
  # May return false due to elastic logging
442
519
  should_log = log.assign(
443
520
  message: message,
@@ -37,11 +37,11 @@ module SemanticLogger
37
37
  end
38
38
 
39
39
  # :nodoc:
40
- def add(severity, message = nil, progname = nil, &block)
40
+ def add(severity, message = nil, progname = nil, &)
41
41
  index = Levels.index(severity)
42
42
  if level_index <= index
43
43
  level = Levels.level(index)
44
- log_internal(level, index, message, progname, &block)
44
+ log_internal(level, index, message, progname, &)
45
45
  true
46
46
  else
47
47
  false
@@ -0,0 +1,34 @@
1
+ module SemanticLogger
2
+ module CoreExt
3
+ # Reopen all appenders in the child process after a fork.
4
+ #
5
+ # Prepended onto `Process.singleton_class` when Semantic Logger is loaded.
6
+ # Enabled by default; opt out with `SemanticLogger.reopen_on_fork = false`.
7
+ #
8
+ # `Process._fork` (Ruby 3.1+) is the single method that every fork path routes
9
+ # through (`Kernel#fork`, `Process.fork`, `IO.popen`, `Kernel#system`, and
10
+ # backticks), so overriding it covers them all with one hook.
11
+ #
12
+ # Note: both `_fork` and `daemon` must be overridden to reliably restart the
13
+ # appender thread, since `Process.daemon` does not route through `_fork`
14
+ # (https://bugs.ruby-lang.org/issues/18911). Do not collapse this down to only
15
+ # `_fork`.
16
+ module Process
17
+ # `_fork` runs in both the parent and the child. It returns 0 in the child
18
+ # and the child's pid in the parent, so only reopen in the child. Reopening
19
+ # in the parent would needlessly recreate its queue and worker thread,
20
+ # risking the loss of messages still on the queue.
21
+ def _fork
22
+ child_pid = super
23
+ SemanticLogger.reopen if child_pid.zero? && SemanticLogger.reopen_on_fork?
24
+ child_pid
25
+ end
26
+
27
+ # `Process.daemon` does not route through `_fork`, so reopen explicitly. Once
28
+ # it returns, the caller is running as the daemon (child) process.
29
+ def daemon(...)
30
+ super.tap { SemanticLogger.reopen if SemanticLogger.reopen_on_fork? }
31
+ end
32
+ end
33
+ end
34
+ end
@@ -2,7 +2,21 @@ require "time"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Base
5
- attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision
5
+ attr_accessor :log, :logger, :time_format, :log_host, :log_application, :log_environment, :precision,
6
+ :escape_control_chars
7
+
8
+ # Printable escapes for the most common control characters. Any other
9
+ # control character is escaped to its hexadecimal `\xHH` form by
10
+ # #escape_control_characters.
11
+ CONTROL_CHAR_ESCAPES = {
12
+ "\n" => "\\n",
13
+ "\r" => "\\r",
14
+ "\t" => "\\t",
15
+ "\e" => "\\e"
16
+ }.freeze
17
+
18
+ # Matches C0 control characters (including newlines and the ANSI escape) and DEL.
19
+ CONTROL_CHARS = /[\x00-\x1f\x7f]/
6
20
 
7
21
  # Time precision varies by Ruby interpreter
8
22
  # JRuby 9.1.8.0 supports microseconds
@@ -34,16 +48,27 @@ module SemanticLogger
34
48
  # precision: [Integer]
35
49
  # How many fractional digits to log times with.
36
50
  # Default: PRECISION (6, except on older JRuby, where 3)
51
+ # escape_control_chars: [Boolean]
52
+ # Replace control characters (newlines, the ANSI escape, etc.) in
53
+ # untrusted log data (message, tags, named tags, and exception
54
+ # message) with a printable escaped form, e.g. "\n".
55
+ # This prevents log forging and terminal escape-sequence injection
56
+ # when logging data that may contain attacker-controlled content.
57
+ # Note: Has no effect on structured formatters such as :json, which
58
+ # already escape control characters via JSON encoding.
59
+ # Default: false (preserve newlines and ANSI colors in text output)
37
60
  def initialize(time_format: nil,
38
61
  log_host: true,
39
62
  log_application: true,
40
63
  log_environment: true,
41
- precision: PRECISION)
42
- @time_format = time_format || self.class.build_time_format(precision)
43
- @log_host = log_host
44
- @log_application = log_application
45
- @log_environment = log_environment
46
- @precision = precision
64
+ precision: PRECISION,
65
+ escape_control_chars: false)
66
+ @time_format = time_format || self.class.build_time_format(precision)
67
+ @log_host = log_host
68
+ @log_application = log_application
69
+ @log_environment = log_environment
70
+ @precision = precision
71
+ @escape_control_chars = escape_control_chars
47
72
  end
48
73
 
49
74
  # Return default time format string
@@ -68,6 +93,20 @@ module SemanticLogger
68
93
 
69
94
  private
70
95
 
96
+ # When `escape_control_chars` is enabled, return a copy of the supplied
97
+ # value with any control characters replaced by a printable escaped form
98
+ # so that untrusted log data cannot forge log entries or inject terminal
99
+ # escape sequences. Otherwise the value is returned unchanged.
100
+ #
101
+ # Note: This escapes (preserves) control characters, and is opt-in. It is
102
+ # distinct from Log#cleansed_message, which unconditionally *strips* ANSI
103
+ # colorization from the message for structured (JSON/Loki) output.
104
+ def escape_control_characters(value)
105
+ return value unless escape_control_chars && value
106
+
107
+ value.to_s.gsub(CONTROL_CHARS) { |char| CONTROL_CHAR_ESCAPES[char] || format("\\x%02x", char.ord) }
108
+ end
109
+
71
110
  # Return the Time as a formatted string
72
111
  def format_time(time)
73
112
  time = time.dup
@@ -85,7 +85,10 @@ module SemanticLogger
85
85
  end
86
86
 
87
87
  def tags
88
- "[#{color}#{log.tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]" if log.tags && !log.tags.empty?
88
+ return if log.tags.nil? || log.tags.empty?
89
+
90
+ tags = log.tags.map { |tag| escape_control_characters(tag) }
91
+ "[#{color}#{tags.join("#{color_map.clear}] [#{color}")}#{color_map.clear}]"
89
92
  end
90
93
 
91
94
  # Named Tags
@@ -94,7 +97,9 @@ module SemanticLogger
94
97
  return if named_tags.nil? || named_tags.empty?
95
98
 
96
99
  list = []
97
- named_tags.each_pair { |name, value| list << "#{color}#{name}: #{value}#{color_map.clear}" }
100
+ named_tags.each_pair do |name, value|
101
+ list << "#{color}#{escape_control_characters(name)}: #{escape_control_characters(value)}#{color_map.clear}"
102
+ end
98
103
  "{#{list.join(', ')}}"
99
104
  end
100
105
 
@@ -123,7 +128,8 @@ module SemanticLogger
123
128
  def exception
124
129
  return unless log.exception
125
130
 
126
- "-- Exception: #{color}#{log.exception.class}: #{log.exception.message}#{color_map.clear}\n#{log.backtrace_to_s}"
131
+ "-- Exception: #{color}#{log.exception.class}: " \
132
+ "#{escape_control_characters(log.exception.message)}#{color_map.clear}\n#{log.backtrace_to_s}"
127
133
  end
128
134
 
129
135
  def call(log, logger)
@@ -32,7 +32,9 @@ module SemanticLogger
32
32
 
33
33
  # Tags
34
34
  def tags
35
- "[#{log.tags.join('] [')}]" if log.tags && !log.tags.empty?
35
+ return if log.tags.nil? || log.tags.empty?
36
+
37
+ "[#{log.tags.map { |tag| escape_control_characters(tag) }.join('] [')}]"
36
38
  end
37
39
 
38
40
  # Named Tags
@@ -41,7 +43,9 @@ module SemanticLogger
41
43
  return if named_tags.nil? || named_tags.empty?
42
44
 
43
45
  list = []
44
- named_tags.each_pair { |name, value| list << "#{name}: #{value}" }
46
+ named_tags.each_pair do |name, value|
47
+ list << "#{escape_control_characters(name)}: #{escape_control_characters(value)}"
48
+ end
45
49
  "{#{list.join(', ')}}"
46
50
  end
47
51
 
@@ -57,7 +61,7 @@ module SemanticLogger
57
61
 
58
62
  # Log message
59
63
  def message
60
- "-- #{log.message}" if log.message
64
+ "-- #{escape_control_characters(log.message)}" if log.message
61
65
  end
62
66
 
63
67
  # Payload
@@ -70,7 +74,10 @@ module SemanticLogger
70
74
 
71
75
  # Exception
72
76
  def exception
73
- "-- Exception: #{log.exception.class}: #{log.exception.message}\n#{log.backtrace_to_s}" if log.exception
77
+ return unless log.exception
78
+
79
+ "-- Exception: #{log.exception.class}: #{escape_control_characters(log.exception.message)}\n" \
80
+ "#{log.backtrace_to_s}"
74
81
  end
75
82
 
76
83
  # Default text log format
@@ -0,0 +1,151 @@
1
+ require "json"
2
+ module SemanticLogger
3
+ module Formatters
4
+ # Formatter conforming to the Elastic Common Schema (ECS).
5
+ #
6
+ # Emits log events using the nested field names defined by ECS so that they
7
+ # integrate cleanly with Filebeat and the Elastic stack (Elasticsearch,
8
+ # Kibana) without requiring an ingest pipeline to rename fields.
9
+ #
10
+ # Usage:
11
+ # SemanticLogger.add_appender(io: $stdout, formatter: :ecs)
12
+ #
13
+ # # Route the payload, metric, and other SemanticLogger-specific data into
14
+ # # a custom top-level namespace (default "semantic_logger"):
15
+ # SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: "my_app"}})
16
+ #
17
+ # # Or merge the payload directly into ECS `labels` instead of a namespace:
18
+ # SemanticLogger.add_appender(io: $stdout, formatter: {ecs: {namespace: nil}})
19
+ #
20
+ # == Field mapping (SemanticLogger -> ECS 8.x)
21
+ # time -> @timestamp (ISO-8601)
22
+ # level -> log.level
23
+ # name -> log.logger
24
+ # file_name / line -> log.origin.file.name / log.origin.file.line
25
+ # message -> message
26
+ # thread_name -> process.thread.name
27
+ # pid -> process.pid
28
+ # host -> host.hostname
29
+ # application -> service.name
30
+ # environment -> service.environment
31
+ # exception -> error.type / error.message / error.stack_trace
32
+ # duration -> event.duration (nanoseconds, as required by ECS)
33
+ # tags -> tags (ECS top-level array)
34
+ # named_tags -> labels.* (scalar key/value pairs)
35
+ # payload -> <namespace>.* (or labels.* when namespace is nil)
36
+ # metric/metric_amount -> <namespace>.metric / <namespace>.metric_amount
37
+ #
38
+ # == Reference
39
+ # * https://www.elastic.co/docs/reference/ecs
40
+ # * https://www.elastic.co/docs/reference/ecs/ecs-custom-fields-in-ecs
41
+ class Ecs < Raw
42
+ # ECS version this formatter targets.
43
+ ECS_VERSION = "8.11.0".freeze
44
+
45
+ # namespace: [String|Symbol|nil]
46
+ # Top-level field set used to hold SemanticLogger-specific data that has
47
+ # no native ECS home (payload, metric, metric_amount). A proper-noun
48
+ # namespace is guaranteed never to collide with a current or future ECS
49
+ # field. Set to nil to merge the payload into ECS `labels` instead.
50
+ # Default: "semantic_logger"
51
+ attr_reader :namespace
52
+
53
+ def initialize(namespace: "semantic_logger", time_format: :iso_8601, time_key: :timestamp, **args)
54
+ @namespace = namespace&.to_sym
55
+
56
+ super(time_format: time_format, time_key: time_key, **args)
57
+ end
58
+
59
+ # Returns the log event as a single line of ECS-formatted JSON, so it can
60
+ # be written to stdout / a file and shipped by Filebeat or Elastic Agent.
61
+ def call(log, logger)
62
+ Utils.to_json(ecs_hash(super))
63
+ end
64
+
65
+ # Returns a batch of log events as a single JSON array.
66
+ def batch(logs, logger)
67
+ "[#{logs.map { |log| call(log, logger) }.join(',')}]"
68
+ end
69
+
70
+ private
71
+
72
+ # Remap the flat hash built by Raw#call into the nested ECS field layout.
73
+ def ecs_hash(hash)
74
+ result = base(hash)
75
+ result[:process] = process(hash)
76
+ result[:host] = {hostname: hash[:host]} if hash[:host]
77
+ add_service(result, hash)
78
+ add_origin(result, hash)
79
+ add_event(result, hash)
80
+ result[:tags] = hash[:tags] if hash[:tags]
81
+ add_error(result, hash)
82
+ add_extras(result, hash)
83
+ result
84
+ end
85
+
86
+ def base(hash)
87
+ log = {level: hash[:level].to_s}
88
+ log[:logger] = hash[:name] if hash[:name]
89
+ {
90
+ "@timestamp": hash[:timestamp],
91
+ message: hash[:message],
92
+ ecs: {version: ECS_VERSION},
93
+ log: log
94
+ }
95
+ end
96
+
97
+ def process(hash)
98
+ result = {pid: hash[:pid]}
99
+ result[:thread] = {name: hash[:thread].to_s} if hash[:thread]
100
+ result
101
+ end
102
+
103
+ def add_service(result, hash)
104
+ service = {}
105
+ service[:name] = hash[:application] if hash[:application]
106
+ service[:environment] = hash[:environment] if hash[:environment]
107
+ result[:service] = service unless service.empty?
108
+ end
109
+
110
+ def add_origin(result, hash)
111
+ return unless hash[:file]
112
+
113
+ result[:log][:origin] = {file: {name: hash[:file], line: hash[:line]}.compact}
114
+ end
115
+
116
+ # ECS event.duration is measured in nanoseconds.
117
+ def add_event(result, hash)
118
+ return unless hash[:duration_ms]
119
+
120
+ result[:event] = {duration: (hash[:duration_ms] * 1_000_000).round}
121
+ end
122
+
123
+ def add_error(result, hash)
124
+ return unless hash[:exception]
125
+
126
+ result[:error] = {
127
+ type: hash[:exception][:name],
128
+ message: hash[:exception][:message],
129
+ stack_trace: Array(hash[:exception][:stack_trace]).join("\n")
130
+ }
131
+ end
132
+
133
+ # Place SemanticLogger-specific data (payload, metric) that has no native
134
+ # ECS home into the configured namespace, plus named_tags into labels.
135
+ def add_extras(result, hash)
136
+ labels = hash[:named_tags].is_a?(Hash) ? hash[:named_tags].dup : {}
137
+
138
+ extra = {}
139
+ extra[:payload] = hash[:payload] if hash[:payload]
140
+ extra[:metric] = hash[:metric] if hash[:metric]
141
+ extra[:metric_amount] = hash[:metric_amount] if hash[:metric_amount]
142
+
143
+ unless extra.empty?
144
+ namespace ? result[namespace] = extra : labels.merge!(extra)
145
+ end
146
+
147
+ result[:labels] = labels unless labels.empty?
148
+ end
149
+ end
150
+ end
151
+ end
@@ -7,9 +7,9 @@ module SemanticLogger
7
7
  class Fluentd < Json
8
8
  attr_reader :need_process_info
9
9
 
10
- def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
10
+ def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, log_host: false, **args)
11
11
  @need_process_info = need_process_info
12
- super(time_format: time_format, time_key: time_key, **args)
12
+ super(time_format: time_format, time_key: time_key, log_host: log_host, **args)
13
13
  end
14
14
 
15
15
  def level
@@ -17,8 +17,19 @@ module SemanticLogger
17
17
  hash["severity_index"] = log.level_index
18
18
  end
19
19
 
20
- def process_info
21
- # Ignore fields: pid, thread, file and line by default
20
+ # Ignore process fields: pid, thread, file and line by default.
21
+ # These are rarely useful under Fluentd (e.g. containerized processes
22
+ # usually have pid 1), so they are only included when explicitly requested
23
+ # via `need_process_info: true`.
24
+ def pid
25
+ super if need_process_info
26
+ end
27
+
28
+ def thread_name
29
+ super if need_process_info
30
+ end
31
+
32
+ def file_name_and_line
22
33
  super if need_process_info
23
34
  end
24
35
  end
@@ -9,7 +9,12 @@ module SemanticLogger
9
9
 
10
10
  # Returns log messages in JSON format
11
11
  def call(log, logger)
12
- super.to_json
12
+ Utils.to_json(super)
13
+ end
14
+
15
+ # Returns a batch of log messages as a single JSON array.
16
+ def batch(logs, logger)
17
+ "[#{logs.map { |log| call(log, logger) }.join(',')}]"
13
18
  end
14
19
  end
15
20
  end
@@ -8,11 +8,13 @@ module SemanticLogger
8
8
  # :timestamp, :level, :name, :message, :duration, :tags, :named_tags
9
9
  #
10
10
  # E.g.
11
- # timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast" spaces="second breakfast" double_quotes="\"elevensies\"" single_quotes="'lunch'" tag="success"
11
+ # timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast"
12
+ # spaces="second breakfast" double_quotes="\"elevensies\"" single_quotes="'lunch'" tag="success"
12
13
  #
13
14
  # All timestamps are ISO8601 formatteed
14
15
  # All user supplied values are escaped and surrounded by double quotes to avoid ambiguious message delimeters
15
- # `tags` are treated as keys with boolean values. Tag names are not formatted or validated, ensure you use valid logfmt format for tag names.
16
+ # `tags` are treated as keys with boolean values. Tag names are not formatted or validated;
17
+ # ensure you use valid logfmt format for tag names.
16
18
  # `named_tags` are flattened are merged into the top level message field. Any conflicting fields are overridden.
17
19
  # `payload` values take precedence over `tags` and `named_tags`. Any conflicting fields are overridden.
18
20
  #
@@ -64,9 +66,9 @@ module SemanticLogger
64
66
  flattened = @parsed.map do |key, value|
65
67
  case value
66
68
  when Hash, Array
67
- "#{key}=#{value.to_s.to_json}"
69
+ "#{key}=#{Utils.to_json(value.to_s)}"
68
70
  else
69
- "#{key}=#{value.to_json}"
71
+ "#{key}=#{Utils.to_json(value)}"
70
72
  end
71
73
  end
72
74