semantic_logger 4.5.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +51 -21
  3. data/Rakefile +7 -7
  4. data/lib/semantic_logger/ansi_colors.rb +0 -10
  5. data/lib/semantic_logger/appender/async.rb +12 -10
  6. data/lib/semantic_logger/appender/async_batch.rb +7 -3
  7. data/lib/semantic_logger/appender/bugsnag.rb +43 -30
  8. data/lib/semantic_logger/appender/elasticsearch.rb +34 -15
  9. data/lib/semantic_logger/appender/elasticsearch_http.rb +4 -4
  10. data/lib/semantic_logger/appender/file.rb +249 -67
  11. data/lib/semantic_logger/appender/graylog.rb +15 -10
  12. data/lib/semantic_logger/appender/honeybadger.rb +3 -3
  13. data/lib/semantic_logger/appender/http.rb +41 -20
  14. data/lib/semantic_logger/appender/io.rb +68 -0
  15. data/lib/semantic_logger/appender/kafka.rb +46 -31
  16. data/lib/semantic_logger/appender/mongodb.rb +6 -6
  17. data/lib/semantic_logger/appender/new_relic.rb +2 -2
  18. data/lib/semantic_logger/appender/rabbitmq.rb +5 -5
  19. data/lib/semantic_logger/appender/sentry.rb +7 -7
  20. data/lib/semantic_logger/appender/sentry_ruby.rb +138 -0
  21. data/lib/semantic_logger/appender/splunk.rb +7 -5
  22. data/lib/semantic_logger/appender/splunk_http.rb +6 -5
  23. data/lib/semantic_logger/appender/syslog.rb +23 -15
  24. data/lib/semantic_logger/appender/tcp.rb +9 -9
  25. data/lib/semantic_logger/appender/udp.rb +2 -2
  26. data/lib/semantic_logger/appender/wrapper.rb +3 -2
  27. data/lib/semantic_logger/appender.rb +62 -65
  28. data/lib/semantic_logger/appenders.rb +36 -53
  29. data/lib/semantic_logger/base.rb +61 -39
  30. data/lib/semantic_logger/formatters/base.rb +16 -6
  31. data/lib/semantic_logger/formatters/color.rb +14 -15
  32. data/lib/semantic_logger/formatters/default.rb +18 -5
  33. data/lib/semantic_logger/formatters/fluentd.rb +7 -18
  34. data/lib/semantic_logger/formatters/json.rb +3 -5
  35. data/lib/semantic_logger/formatters/logfmt.rb +77 -0
  36. data/lib/semantic_logger/formatters/raw.rb +39 -10
  37. data/lib/semantic_logger/formatters/signalfx.rb +14 -21
  38. data/lib/semantic_logger/formatters/syslog.rb +8 -6
  39. data/lib/semantic_logger/formatters/syslog_cee.rb +9 -7
  40. data/lib/semantic_logger/formatters.rb +13 -13
  41. data/lib/semantic_logger/jruby/garbage_collection_logger.rb +4 -2
  42. data/lib/semantic_logger/levels.rb +9 -7
  43. data/lib/semantic_logger/log.rb +58 -73
  44. data/lib/semantic_logger/loggable.rb +8 -1
  45. data/lib/semantic_logger/logger.rb +19 -11
  46. data/lib/semantic_logger/metric/new_relic.rb +3 -3
  47. data/lib/semantic_logger/metric/signalfx.rb +3 -3
  48. data/lib/semantic_logger/metric/statsd.rb +7 -7
  49. data/lib/semantic_logger/processor.rb +9 -7
  50. data/lib/semantic_logger/reporters/minitest.rb +4 -4
  51. data/lib/semantic_logger/semantic_logger.rb +57 -23
  52. data/lib/semantic_logger/subscriber.rb +24 -7
  53. data/lib/semantic_logger/sync.rb +12 -0
  54. data/lib/semantic_logger/sync_processor.rb +58 -0
  55. data/lib/semantic_logger/test/capture_log_events.rb +34 -0
  56. data/lib/semantic_logger/utils.rb +32 -13
  57. data/lib/semantic_logger/version.rb +1 -1
  58. data/lib/semantic_logger.rb +27 -22
  59. metadata +15 -10
@@ -1,37 +1,26 @@
1
- require 'json'
1
+ require "json"
2
2
 
3
3
  module SemanticLogger
4
4
  module Formatters
5
- # Fluentd is similar to SemanticLogger::Formatters::Json but with log level that are recongnized
5
+ # Fluentd is similar to SemanticLogger::Formatters::Json but with log levels that are recognized
6
6
  # by kubernetes fluentd.
7
7
  class Fluentd < Json
8
8
  attr_reader :need_process_info
9
9
 
10
- def initialize(log_host: true, log_application: true, need_process_info: false)
10
+ def initialize(time_format: :rfc_3339, time_key: :time, need_process_info: false, **args)
11
11
  @need_process_info = need_process_info
12
- super(log_host: log_host, log_application: log_application, time_key: 'time', time_format: :rfc_3339)
12
+ super(time_format: time_format, time_key: time_key, **args)
13
13
  end
14
14
 
15
- def severity
16
- hash['severity'] = log.level
17
- hash['severity_index'] = log.level_index
15
+ def level
16
+ hash["severity"] = log.level
17
+ hash["severity_index"] = log.level_index
18
18
  end
19
19
 
20
20
  def process_info
21
21
  # Ignore fields: pid, thread, file and line by default
22
22
  super() if need_process_info
23
23
  end
24
-
25
- def call(log, logger)
26
- self.hash = {}
27
- self.log = log
28
- self.logger = logger
29
-
30
- host; application; time; severity; process_info; duration; tags; named_tags; name; message; payload; exception; metric
31
- hash
32
-
33
- hash.to_json
34
- end
35
24
  end
36
25
  end
37
26
  end
@@ -1,12 +1,10 @@
1
- require 'json'
1
+ require "json"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Json < Raw
5
5
  # Default JSON time format is ISO8601
6
- def initialize(time_format: :iso_8601, log_host: true, log_application: true, time_key: :timestamp,
7
- precision: PRECISION)
8
- super(time_format: time_format, log_host: log_host, log_application: log_application, time_key: time_key,
9
- precision: precision)
6
+ def initialize(time_format: :iso_8601, time_key: :timestamp, **args)
7
+ super(time_format: time_format, time_key: time_key, **args)
10
8
  end
11
9
 
12
10
  # Returns log messages in JSON format
@@ -0,0 +1,77 @@
1
+ require "json"
2
+
3
+ module SemanticLogger
4
+ module Formatters
5
+ # Produces logfmt formatted messages
6
+ #
7
+ # The following fields are extracted from the raw log and included in the formatted message:
8
+ # :timestamp, :level, :name, :message, :duration, :tags, :named_tags
9
+ #
10
+ # E.g.
11
+ # timestamp="2020-07-20T08:32:05.375276Z" level=info name="DefaultTest" base="breakfast" spaces="second breakfast" double_quotes="\"elevensies\"" single_quotes="'lunch'" tag="success"
12
+ #
13
+ # All timestamps are ISO8601 formatteed
14
+ # All user supplied values are escaped and surrounded by double quotes to avoid ambiguious message delimeters
15
+ # `tags` are treated as keys with boolean values. Tag names are not formatted or validated, ensure you use valid logfmt format for tag names.
16
+ # `named_tags` are flattened are merged into the top level message field. Any conflicting fields are overridden.
17
+ # `payload` values take precedence over `tags` and `named_tags`. Any conflicting fields are overridden.
18
+ #
19
+ # Futher Reading https://brandur.org/logfmt
20
+ class Logfmt < Raw
21
+ def initialize(time_format: :iso_8601, time_key: :timestamp, **args)
22
+ super(time_format: time_format, time_key: time_key, **args)
23
+ end
24
+
25
+ def call(log, logger)
26
+ @raw = super(log, logger)
27
+
28
+ raw_to_logfmt
29
+ end
30
+
31
+ private
32
+
33
+ def raw_to_logfmt
34
+ @parsed = @raw.slice(time_key, :level, :name, :message, :duration).merge(tag: "success")
35
+ handle_tags
36
+ handle_payload
37
+ handle_exception
38
+
39
+ flatten_log
40
+ end
41
+
42
+ def handle_tags
43
+ tags = @raw.fetch(:tags){ [] }
44
+ .each_with_object({}){ |tag, accum| accum[tag] = true }
45
+
46
+ @parsed = @parsed.merge(tags)
47
+ .merge(@raw.fetch(:named_tags){ {} })
48
+ end
49
+
50
+ def handle_payload
51
+ return unless @raw.key? :payload
52
+
53
+ @parsed = @parsed.merge(@raw[:payload])
54
+ end
55
+
56
+ def handle_exception
57
+ return unless @raw.key? :exception
58
+
59
+ @parsed[:tag] = "exception"
60
+ @parsed = @parsed.merge(@raw[:exception])
61
+ end
62
+
63
+ def flatten_log
64
+ flattened = @parsed.map do |key, value|
65
+ case value
66
+ when Hash, Array
67
+ "#{key}=#{value.to_s.to_json}"
68
+ else
69
+ "#{key}=#{value.to_json}"
70
+ end
71
+ end
72
+
73
+ flattened.join(" ")
74
+ end
75
+ end
76
+ end
77
+ end
@@ -1,14 +1,14 @@
1
- require 'json'
1
+ require "json"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Raw < Base
5
5
  # Fields are added by populating this hash.
6
- attr_accessor :hash, :log, :logger, :time_key
6
+ attr_accessor :hash, :time_key
7
7
 
8
8
  # By default Raw formatter does not reformat the time
9
- def initialize(time_format: :none, log_host: true, log_application: true, time_key: :time, precision: PRECISION)
9
+ def initialize(time_format: :none, time_key: :time, **args)
10
10
  @time_key = time_key
11
- super(time_format: time_format, log_host: log_host, log_application: log_application, precision: precision)
11
+ super(time_format: time_format, **args)
12
12
  end
13
13
 
14
14
  # Host name
@@ -18,7 +18,12 @@ module SemanticLogger
18
18
 
19
19
  # Application name
20
20
  def application
21
- hash[:application] = logger.application if log_application && logger.application
21
+ hash[:application] = logger.application if log_application && logger && logger.application
22
+ end
23
+
24
+ # Environment
25
+ def environment
26
+ hash[:environment] = logger.environment if log_environment && logger && logger.environment
22
27
  end
23
28
 
24
29
  # Date & time
@@ -32,11 +37,18 @@ module SemanticLogger
32
37
  hash[:level_index] = log.level_index
33
38
  end
34
39
 
35
- # Process info
36
- def process_info
37
- hash[:pid] = $$
40
+ # Process ID
41
+ def pid
42
+ hash[:pid] = super
43
+ end
44
+
45
+ # Name of the thread that logged the message.
46
+ def thread_name
38
47
  hash[:thread] = log.thread_name
48
+ end
39
49
 
50
+ # Ruby file name and line number that logged the message.
51
+ def file_name_and_line
40
52
  file, line = log.file_name_and_line
41
53
  return unless file
42
54
 
@@ -80,6 +92,7 @@ module SemanticLogger
80
92
  # Exception
81
93
  def exception
82
94
  return unless log.exception
95
+
83
96
  root = hash
84
97
  log.each_exception do |exception, i|
85
98
  name = i.zero? ? :exception : :cause
@@ -88,7 +101,7 @@ module SemanticLogger
88
101
  message: exception.message,
89
102
  stack_trace: exception.backtrace
90
103
  }
91
- root = root[name]
104
+ root = root[name]
92
105
  end
93
106
  end
94
107
 
@@ -104,7 +117,23 @@ module SemanticLogger
104
117
  self.log = log
105
118
  self.logger = logger
106
119
 
107
- host; application; time; level; process_info; duration; tags; named_tags; name; message; payload; exception; metric
120
+ host
121
+ application
122
+ environment
123
+ time
124
+ level
125
+ pid
126
+ thread_name
127
+ file_name_and_line
128
+ duration
129
+ tags
130
+ named_tags
131
+ name
132
+ message
133
+ payload
134
+ exception
135
+ metric
136
+
108
137
  hash
109
138
  end
110
139
  end
@@ -1,49 +1,41 @@
1
- require 'json'
1
+ require "json"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Signalfx < Base
5
- attr_accessor :token, :dimensions, :hash, :log, :logger, :gauge_name, :counter_name, :environment
5
+ attr_accessor :token, :dimensions, :hash, :gauge_name, :counter_name
6
6
 
7
7
  def initialize(token:,
8
8
  dimensions: nil,
9
- log_host: true,
10
- log_application: true,
11
- gauge_name: 'Application.average',
12
- counter_name: 'Application.counter',
13
- environment: true,
14
- precision: PRECISION)
9
+ gauge_name: "Application.average",
10
+ counter_name: "Application.counter",
11
+ time_format: :ms,
12
+ **args)
15
13
 
16
14
  @token = token
17
15
  @dimensions = dimensions.map(&:to_sym) if dimensions
18
16
  @gauge_name = gauge_name
19
17
  @counter_name = counter_name
20
18
 
21
- if environment == true
22
- @environment = defined?(Rails) && Rails.respond_to?(:env) ? Rails.env : ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
23
- elsif environment
24
- @environment = environment
25
- end
26
-
27
- super(time_format: :ms, log_host: log_host, log_application: log_application, precision: precision)
19
+ super(time_format: time_format, **args)
28
20
  end
29
21
 
30
22
  # Create SignalFx friendly metric.
31
23
  # Strip leading '/'
32
24
  # Convert remaining '/' to '.'
33
25
  def metric
34
- name = log.metric.to_s.sub(/\A\/+/, '')
26
+ name = log.metric.to_s.sub(%r{\A/+}, "")
35
27
  if log.dimensions
36
- name.tr!('/', '.')
28
+ name.tr!("/", ".")
37
29
  hash[:metric] = name
38
30
  else
39
31
  # Extract class and action from metric name
40
- names = name.split('/')
32
+ names = name.split("/")
41
33
  h = (hash[:dimensions] ||= {})
42
34
  if names.size > 1
43
35
  h[:action] = names.pop
44
- h[:class] = names.join('::')
36
+ h[:class] = names.join("::")
45
37
  else
46
- h[:class] = 'Unknown'
38
+ h[:class] = "Unknown"
47
39
  h[:action] = names.first || log.metric
48
40
  end
49
41
 
@@ -75,12 +67,13 @@ module SemanticLogger
75
67
  name = name.to_sym
76
68
  value = value.to_s
77
69
  next if value.empty?
70
+
78
71
  h[name] = value if dimensions&.include?(name)
79
72
  end
80
73
  end
81
74
  h[:host] = logger.host if log_host && logger.host
82
75
  h[:application] = logger.application if log_application && logger.application
83
- h[:environment] = environment if environment
76
+ h[:environment] = logger.environment if log_environment && logger.environment
84
77
  end
85
78
 
86
79
  # Returns [Hash] log message in Signalfx format.
@@ -1,13 +1,14 @@
1
1
  begin
2
- require 'syslog_protocol'
2
+ require "syslog_protocol"
3
3
  rescue LoadError
4
- raise 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
4
+ raise LoadError,
5
+ 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
5
6
  end
6
7
 
7
8
  module SemanticLogger
8
9
  module Formatters
9
10
  class Syslog < Default
10
- attr_accessor :level_map, :facility
11
+ attr_accessor :level_map, :facility, :max_size
11
12
 
12
13
  # Default level map for every log level
13
14
  #
@@ -50,9 +51,10 @@ module SemanticLogger
50
51
  # Example:
51
52
  # # Change the warn level to LOG_NOTICE level instead of a the default of LOG_WARNING.
52
53
  # SemanticLogger.add_appender(appender: :syslog, level_map: {warn: ::Syslog::LOG_NOTICE})
53
- def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new)
54
+ def initialize(facility: ::Syslog::LOG_USER, level_map: LevelMap.new, max_size: Integer)
54
55
  @facility = facility
55
56
  @level_map = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(level_map)
57
+ @max_size = max_size
56
58
  super()
57
59
  end
58
60
 
@@ -73,11 +75,11 @@ module SemanticLogger
73
75
  packet = SyslogProtocol::Packet.new
74
76
  packet.hostname = logger.host
75
77
  packet.facility = facility
76
- packet.tag = logger.application.delete(' ')
78
+ packet.tag = logger.application.delete(" ")
77
79
  packet.content = message
78
80
  packet.time = log.time
79
81
  packet.severity = level_map[log.level]
80
- packet.to_s
82
+ packet.assemble(@max_size)
81
83
  end
82
84
  end
83
85
  end
@@ -1,13 +1,14 @@
1
1
  begin
2
- require 'syslog_protocol'
2
+ require "syslog_protocol"
3
3
  rescue LoadError
4
- raise 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
4
+ raise LoadError,
5
+ 'Gem syslog_protocol is required for remote logging using the Syslog protocol. Please add the gem "syslog_protocol" to your Gemfile.'
5
6
  end
6
7
 
7
8
  module SemanticLogger
8
9
  module Formatters
9
10
  class SyslogCee < Raw
10
- attr_accessor :level_map, :facility
11
+ attr_accessor :level_map, :facility, :max_size
11
12
 
12
13
  # CEE JSON Syslog format
13
14
  # Untested prototype code. Based on documentation only.
@@ -22,10 +23,11 @@ module SemanticLogger
22
23
  #
23
24
  # Example:
24
25
  # # Log via udp to a remote syslog server on host: `server1` and port `8514`, using the CEE format.
25
- # SemanticLogger.add_appender(appender: :syslog, formatter: syslog_cee, url: 'udp://server1:8514')
26
- def initialize(facility: ::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new)
26
+ # SemanticLogger.add_appender(appender: :syslog, formatter: :syslog_cee, url: 'udp://server1:8514')
27
+ def initialize(facility: ::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new, max_size: Integer)
27
28
  @facility = facility
28
29
  @level_map = level_map.is_a?(SemanticLogger::Formatters::Syslog::LevelMap) ? level_map : SemanticLogger::Formatters::Syslog::LevelMap.new(level_map)
30
+ @max_size = max_size
29
31
  super()
30
32
  end
31
33
 
@@ -45,11 +47,11 @@ module SemanticLogger
45
47
  packet = SyslogProtocol::Packet.new
46
48
  packet.hostname = logger.host
47
49
  packet.facility = facility
48
- packet.tag = logger.application.delete(' ')
50
+ packet.tag = logger.application.delete(" ")
49
51
  packet.content = message
50
52
  packet.time = log.time
51
53
  packet.severity = level_map[log.level]
52
- packet.to_s
54
+ packet.assemble(@max_size)
53
55
  end
54
56
  end
55
57
  end
@@ -1,16 +1,16 @@
1
1
  module SemanticLogger
2
2
  module Formatters
3
- # @formatter:off
4
- autoload :Base, 'semantic_logger/formatters/base'
5
- autoload :Color, 'semantic_logger/formatters/color'
6
- autoload :Default, 'semantic_logger/formatters/default'
7
- autoload :Json, 'semantic_logger/formatters/json'
8
- autoload :Raw, 'semantic_logger/formatters/raw'
9
- autoload :OneLine, 'semantic_logger/formatters/one_line'
10
- autoload :Signalfx, 'semantic_logger/formatters/signalfx'
11
- autoload :Syslog, 'semantic_logger/formatters/syslog'
12
- autoload :Fluentd, 'semantic_logger/formatters/fluentd'
13
- # @formatter:on
3
+ autoload :Base, "semantic_logger/formatters/base"
4
+ autoload :Color, "semantic_logger/formatters/color"
5
+ autoload :Default, "semantic_logger/formatters/default"
6
+ autoload :Json, "semantic_logger/formatters/json"
7
+ autoload :Raw, "semantic_logger/formatters/raw"
8
+ autoload :OneLine, "semantic_logger/formatters/one_line"
9
+ autoload :Signalfx, "semantic_logger/formatters/signalfx"
10
+ autoload :Syslog, "semantic_logger/formatters/syslog"
11
+ autoload :Fluentd, "semantic_logger/formatters/fluentd"
12
+ autoload :Logfmt, "semantic_logger/formatters/logfmt"
13
+ autoload :SyslogCee, "semantic_logger/formatters/syslog_cee"
14
14
 
15
15
  # Return formatter that responds to call.
16
16
  #
@@ -22,10 +22,10 @@ module SemanticLogger
22
22
  # - Any object that responds to :call
23
23
  def self.factory(formatter)
24
24
  if formatter.is_a?(Symbol)
25
- SemanticLogger::Utils.constantize_symbol(formatter, 'SemanticLogger::Formatters').new
25
+ SemanticLogger::Utils.constantize_symbol(formatter, "SemanticLogger::Formatters").new
26
26
  elsif formatter.is_a?(Hash) && formatter.size.positive?
27
27
  fmt, options = formatter.first
28
- SemanticLogger::Utils.constantize_symbol(fmt.to_sym, 'SemanticLogger::Formatters').new(options)
28
+ SemanticLogger::Utils.constantize_symbol(fmt.to_sym, "SemanticLogger::Formatters").new(**options)
29
29
  elsif formatter.respond_to?(:call)
30
30
  formatter
31
31
  else
@@ -12,7 +12,9 @@ module SemanticLogger
12
12
  # Must leave the method name as-is so that it can be found by Java
13
13
  def handleNotification(notification, _)
14
14
  # Only care about GARBAGE_COLLECTION_NOTIFICATION notifications
15
- return unless notification.get_type == Java::ComSunManagement::GarbageCollectionNotificationInfo::GARBAGE_COLLECTION_NOTIFICATION
15
+ unless notification.get_type == Java::ComSunManagement::GarbageCollectionNotificationInfo::GARBAGE_COLLECTION_NOTIFICATION
16
+ return
17
+ end
16
18
 
17
19
  info = Java::ComSunManagement::GarbageCollectionNotificationInfo.from(notification.user_data)
18
20
  gc_info = info.gc_info
@@ -20,7 +22,7 @@ module SemanticLogger
20
22
 
21
23
  return unless duration >= @min_microseconds
22
24
 
23
- SemanticLogger['GarbageCollector'].measure_warn(
25
+ SemanticLogger["GarbageCollector"].measure_warn(
24
26
  "Garbage Collection completed: #{info.gc_name} ##{gc_info.id}",
25
27
  duration: duration.to_f / 1000
26
28
  )
@@ -16,17 +16,19 @@ module SemanticLogger
16
16
  LEVELS.index(level)
17
17
  elsif level.is_a?(Integer) && defined?(::Logger::Severity)
18
18
  # Mapping of Rails and Ruby Logger levels to SemanticLogger levels
19
- @map_levels ||= begin
20
- levels = []
21
- ::Logger::Severity.constants.each do |constant|
22
- levels[::Logger::Severity.const_get(constant)] =
23
- LEVELS.find_index(constant.downcase.to_sym) || LEVELS.find_index(:error)
19
+ @map_levels ||=
20
+ begin
21
+ levels = []
22
+ ::Logger::Severity.constants.each do |constant|
23
+ levels[::Logger::Severity.const_get(constant)] =
24
+ LEVELS.find_index(constant.downcase.to_sym) || LEVELS.find_index(:error)
25
+ end
26
+ levels
24
27
  end
25
- levels
26
- end
27
28
  @map_levels[level]
28
29
  end
29
30
  raise "Invalid level:#{level.inspect} being requested. Must be one of #{LEVELS.inspect}" unless index
31
+
30
32
  index
31
33
  end
32
34