semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -2,6 +2,10 @@ require "json"
2
2
  module SemanticLogger
3
3
  module Formatters
4
4
  class Signalfx < Base
5
+ # Text of a dimension value that is blank (see #blank_value?).
6
+ BLANK_VALUE = /\A[[:space:]]*\z/
7
+ private_constant :BLANK_VALUE
8
+
5
9
  attr_accessor :token, :dimensions, :hash, :gauge_name, :counter_name
6
10
 
7
11
  def initialize(token:,
@@ -21,9 +25,14 @@ module SemanticLogger
21
25
  # Create SignalFx friendly metric.
22
26
  # Strip leading '/'
23
27
  # Convert remaining '/' to '.'
28
+ # A name that is not valid UTF-8, such as one with invalid bytes or in UTF-16, which matching
29
+ # it raises for, is converted to valid UTF-8 first (see Utils.encode_utf8_string).
30
+ #
31
+ # A metric with dimensions that are not blank (see #present_dimensions) is sent under its own name.
32
+ # Any other is sent as the gauge or the counter, with the class and action taken from its name.
24
33
  def metric
25
- name = log.metric.to_s.sub(%r{\A/+}, "")
26
- if log.dimensions
34
+ name = Utils.encode_utf8_string(log.metric.to_s).sub(%r{\A/+}, "")
35
+ if metric_dimensions
27
36
  name.tr!("/", ".")
28
37
  hash[:metric] = name
29
38
  else
@@ -53,21 +62,19 @@ module SemanticLogger
53
62
  hash[:value] = log.metric_amount || log.duration || 1
54
63
  end
55
64
 
56
- # Dimensions for this metric
65
+ # Dimensions for this metric: those of the log entry that are not blank, or when it has none, the
66
+ # named tags listed in the dimensions option whose values are not blank (see #blank_value?).
57
67
  def format_dimensions
58
68
  h = (hash[:dimensions] ||= {})
59
- if log.dimensions
60
- log.dimensions.each_pair do |name, value|
61
- value = value.to_s
62
- h[name] = value unless value.empty?
63
- end
64
- else
69
+ if metric_dimensions
70
+ h.merge!(metric_dimensions)
71
+ elsif dimensions
65
72
  log.named_tags.each_pair do |name, value|
66
- name = name.to_sym
67
- value = value.to_s
68
- next if value.empty?
73
+ name = name.to_sym
74
+ next unless dimensions.include?(name)
69
75
 
70
- h[name] = value if dimensions&.include?(name)
76
+ value = value.to_s
77
+ h[name] = value unless blank_value?(value)
71
78
  end
72
79
  end
73
80
  h[:host] = logger.host if log_host && logger.host
@@ -91,7 +98,7 @@ module SemanticLogger
91
98
  if log.duration
92
99
  data[:gauge] = [hash]
93
100
  # Also send a count metric whenever it is a gauge so that it can be counted.
94
- unless log.dimensions
101
+ unless metric_dimensions
95
102
  count_hash = hash.dup
96
103
  count_hash[:value] = log.metric_amount || 1
97
104
  count_hash[:metric] = counter_name
@@ -125,7 +132,7 @@ module SemanticLogger
125
132
  add_gauge(gauges, hash)
126
133
 
127
134
  # Also send a count metric whenever it is a gauge so that it can be counted.
128
- unless log.dimensions
135
+ unless metric_dimensions
129
136
  count_hash = hash.dup
130
137
  count_hash[:value] = log.metric_amount || 1
131
138
  count_hash[:metric] = counter_name
@@ -141,8 +148,41 @@ module SemanticLogger
141
148
  Utils.to_json(data)
142
149
  end
143
150
 
151
+ # The log entry to format. Also keeps the dimensions of its metric that are sent (see
152
+ # #present_dimensions), which formatting the entry asks for several times.
153
+ def log=(log)
154
+ super
155
+ @metric_dimensions = log && present_dimensions
156
+ end
157
+
144
158
  private
145
159
 
160
+ # The dimensions of the entry's metric that are sent, or nil when there are none.
161
+ attr_reader :metric_dimensions
162
+
163
+ # Returns [Hash] the dimensions of the entry's metric, as Log#dimensions? tells, with each value as
164
+ # text, leaving out those that are blank (see #blank_value?). Returns nil when there are none, or only
165
+ # blank ones, such as `{state: ""}`, so that the metric is sent as a metric without dimensions is:
166
+ # as the gauge or the counter, with the class and action taken from its name. Log#dimensions?
167
+ # counts such dimensions, which the formatters that write the entry's data write as they are.
168
+ def present_dimensions
169
+ return unless log.dimensions?
170
+
171
+ present = {}
172
+ log.dimensions.each_pair do |name, value|
173
+ value = value.to_s
174
+ present[name] = value unless blank_value?(value)
175
+ end
176
+ present unless present.empty?
177
+ end
178
+
179
+ # Returns [true|false] whether the text of a dimension value is blank: empty, as for nil, or only
180
+ # whitespace, which includes Unicode whitespace such as a no-break space, but not NUL. The text is
181
+ # converted to valid UTF-8 first (see Utils.encode_utf8_string): matching raises for UTF-8 text
182
+ # with invalid bytes, or text that is not ASCII-compatible, such as UTF-16, and would not find the
183
+ # Unicode whitespace in text in an encoding such as binary or Shift_JIS.
184
+ def blank_value?(text) = BLANK_VALUE.match?(Utils.encode_utf8_string(text))
185
+
146
186
  def add_gauge(gauges, metric)
147
187
  gauges << metric
148
188
  end
@@ -1,6 +1,8 @@
1
1
  module SemanticLogger
2
2
  module Formatters
3
3
  class Syslog < Default
4
+ include SyslogHeader
5
+
4
6
  attr_accessor :max_size
5
7
  attr_reader :facility, :level_map
6
8
 
@@ -173,9 +175,9 @@ module SemanticLogger
173
175
  # Create Syslog Packet
174
176
  def create_syslog_packet(message)
175
177
  packet = SyslogProtocol::Packet.new
176
- packet.hostname = logger.host
178
+ packet.hostname = syslog_hostname(logger.host)
177
179
  packet.facility = @facility_number
178
- packet.tag = logger.application.delete(" ")
180
+ packet.tag = syslog_tag(logger.application)
179
181
  packet.content = message
180
182
  packet.time = log.time
181
183
  packet.severity = level_map[log.level]
@@ -1,6 +1,8 @@
1
1
  module SemanticLogger
2
2
  module Formatters
3
3
  class SyslogCee < Raw
4
+ include SyslogHeader
5
+
4
6
  attr_accessor :max_size
5
7
  attr_reader :facility, :level_map
6
8
 
@@ -20,17 +22,21 @@ module SemanticLogger
20
22
  # Maximum size of each syslog packet, in bytes.
21
23
  # Default: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE (1024)
22
24
  #
25
+ # Any other options are those of SemanticLogger::Formatters::Raw, such as `log_host` or
26
+ # `log_exception_details`. Its time options (`time_format`, `time_key`, and `precision`)
27
+ # have no effect, since the time is in the header of the syslog packet.
28
+ #
23
29
  # Example:
24
30
  # # Log via udp to a remote syslog server on host: `server1` and port `8514`, using the CEE format.
25
31
  # SemanticLogger.add_appender(appender: :syslog, formatter: :syslog_cee, url: 'udp://server1:8514')
26
32
  def initialize(facility: SemanticLogger::Formatters::Syslog::LOG_USER, level_map: SemanticLogger::Formatters::Syslog::LevelMap.new,
27
- max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE)
33
+ max_size: SemanticLogger::Formatters::Syslog::DEFAULT_MAX_SIZE, **args)
28
34
  SemanticLogger::Formatters::Syslog.require_syslog_protocol
29
35
 
30
36
  self.facility = facility
31
37
  self.level_map = level_map
32
38
  @max_size = max_size
33
- super()
39
+ super(**args)
34
40
  end
35
41
 
36
42
  # Set the facility to one of the ::Syslog::LOG_* facility constants.
@@ -60,9 +66,9 @@ module SemanticLogger
60
66
  # Create Syslog Packet
61
67
  def create_syslog_packet(message)
62
68
  packet = SyslogProtocol::Packet.new
63
- packet.hostname = logger.host
69
+ packet.hostname = syslog_hostname(logger.host)
64
70
  packet.facility = @facility_number
65
- packet.tag = logger.application.delete(" ")
71
+ packet.tag = syslog_tag(logger.application)
66
72
  packet.content = message
67
73
  packet.time = log.time
68
74
  packet.severity = level_map[log.level]
@@ -0,0 +1,49 @@
1
+ module SemanticLogger
2
+ module Formatters
3
+ # The host name and the application name (the TAG) that the formatters building a remote syslog
4
+ # packet (Syslog and SyslogCee) write in its header.
5
+ #
6
+ # The syslog_protocol gem, following RFC 3164, allows only printable ASCII characters other than
7
+ # the space (33 to 126) in either, and up to 32 in the TAG, and raises for any other, which made
8
+ # the formatter lose every entry, such as for an application named "Café", or with a name over
9
+ # 32 characters. So the other characters are left out, as the spaces always were from the TAG,
10
+ # and the TAG is cut to 32 characters. When no character is left, "localhost" or
11
+ # "SemanticLogger" is written instead.
12
+ #
13
+ # Each is worked out once, and again only when the host or application name changes.
14
+ module SyslogHeader
15
+ # The most characters that the TAG of a syslog header can hold.
16
+ MAX_TAG_SIZE = 32
17
+
18
+ # Written in place of a host name with no character left.
19
+ DEFAULT_HOSTNAME = "localhost".encode(Encoding::US_ASCII).freeze
20
+
21
+ # Written in place of an application name with no character left.
22
+ DEFAULT_TAG = "SemanticLogger".encode(Encoding::US_ASCII).freeze
23
+
24
+ private
25
+
26
+ # Returns [String] the host name as the syslog header holds it.
27
+ def syslog_hostname(host) = syslog_header_field(:hostname, host, DEFAULT_HOSTNAME)
28
+
29
+ # Returns [String] the application name as the TAG of the syslog header holds it.
30
+ def syslog_tag(application) = syslog_header_field(:tag, application, DEFAULT_TAG, MAX_TAG_SIZE)
31
+
32
+ # Returns [String] the field cleaned (see SyslogHeader), frozen, from the cache when the text
33
+ # is the one it was worked out from, or holds the same characters, such as an appender's own
34
+ # name that was changed in place, which the cache keeps a frozen copy of. Any other object,
35
+ # other than a Symbol, is read as its #to_s first, so that one changed in place is noticed too.
36
+ def syslog_header_field(field, text, fallback, max_size = nil)
37
+ text = text.to_s unless text.is_a?(String) || text.is_a?(Symbol)
38
+ cached = (@syslog_header_fields ||= {})[field]
39
+ return cached.last if cached && (cached.first.equal?(text) || cached.first == text)
40
+
41
+ value = text.to_s.b.delete("^\x21-\x7E")
42
+ value = value[0, max_size] if max_size
43
+ value = value.empty? ? fallback : value.force_encoding(Encoding::US_ASCII).freeze
44
+ @syslog_header_fields[field] = [text.is_a?(String) && !text.frozen? ? text.dup.freeze : text, value]
45
+ value
46
+ end
47
+ end
48
+ end
49
+ end
@@ -14,6 +14,7 @@ module SemanticLogger
14
14
  autoload :Logfmt, "semantic_logger/formatters/logfmt"
15
15
  autoload :Pattern, "semantic_logger/formatters/pattern"
16
16
  autoload :SyslogCee, "semantic_logger/formatters/syslog_cee"
17
+ autoload :SyslogHeader, "semantic_logger/formatters/syslog_header"
17
18
  autoload :NewRelicLogs, "semantic_logger/formatters/new_relic_logs"
18
19
  autoload :Loki, "semantic_logger/formatters/loki"
19
20