semantic_logger 5.2.0 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/docs/api.md +160 -9
  3. data/docs/appenders.md +461 -34
  4. data/docs/config.md +543 -36
  5. data/docs/log.md +38 -7
  6. data/docs/metrics.md +139 -28
  7. data/docs/operations.md +293 -24
  8. data/docs/rails.md +319 -5
  9. data/docs/security.md +135 -107
  10. data/docs/testing.md +23 -0
  11. data/docs/upgrading.md +644 -1
  12. data/lib/semantic_logger/appender/async.rb +29 -11
  13. data/lib/semantic_logger/appender/bugsnag.rb +7 -2
  14. data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
  15. data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
  16. data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
  17. data/lib/semantic_logger/appender/file.rb +29 -14
  18. data/lib/semantic_logger/appender/graylog.rb +81 -7
  19. data/lib/semantic_logger/appender/honeybadger.rb +7 -1
  20. data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
  21. data/lib/semantic_logger/appender/http.rb +244 -41
  22. data/lib/semantic_logger/appender/io.rb +11 -6
  23. data/lib/semantic_logger/appender/kafka.rb +192 -46
  24. data/lib/semantic_logger/appender/loki.rb +42 -6
  25. data/lib/semantic_logger/appender/mongodb.rb +94 -9
  26. data/lib/semantic_logger/appender/new_relic.rb +2 -3
  27. data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
  28. data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
  29. data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
  30. data/lib/semantic_logger/appender/sentry.rb +2 -3
  31. data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
  32. data/lib/semantic_logger/appender/splunk.rb +4 -4
  33. data/lib/semantic_logger/appender/splunk_http.rb +92 -1
  34. data/lib/semantic_logger/appender/syslog.rb +89 -10
  35. data/lib/semantic_logger/appender/tcp.rb +68 -98
  36. data/lib/semantic_logger/appender/text_output.rb +65 -0
  37. data/lib/semantic_logger/appender/udp.rb +130 -18
  38. data/lib/semantic_logger/appender/wrapper.rb +10 -7
  39. data/lib/semantic_logger/appender.rb +115 -10
  40. data/lib/semantic_logger/appenders.rb +136 -4
  41. data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
  42. data/lib/semantic_logger/base.rb +53 -71
  43. data/lib/semantic_logger/call_site.rb +235 -0
  44. data/lib/semantic_logger/context.rb +222 -0
  45. data/lib/semantic_logger/core_ext/process.rb +20 -5
  46. data/lib/semantic_logger/core_ext/warning.rb +70 -0
  47. data/lib/semantic_logger/deprecation.rb +139 -0
  48. data/lib/semantic_logger/exception_details.rb +4 -3
  49. data/lib/semantic_logger/formatters/base.rb +139 -19
  50. data/lib/semantic_logger/formatters/color.rb +3 -4
  51. data/lib/semantic_logger/formatters/default.rb +35 -7
  52. data/lib/semantic_logger/formatters/ecs.rb +9 -4
  53. data/lib/semantic_logger/formatters/json.rb +9 -0
  54. data/lib/semantic_logger/formatters/logfmt.rb +9 -7
  55. data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
  56. data/lib/semantic_logger/formatters/loki.rb +412 -24
  57. data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
  58. data/lib/semantic_logger/formatters/one_line.rb +44 -10
  59. data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
  60. data/lib/semantic_logger/formatters/pattern.rb +60 -8
  61. data/lib/semantic_logger/formatters/raw.rb +38 -8
  62. data/lib/semantic_logger/formatters/signalfx.rb +55 -15
  63. data/lib/semantic_logger/formatters/syslog.rb +4 -2
  64. data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
  65. data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
  66. data/lib/semantic_logger/formatters.rb +1 -0
  67. data/lib/semantic_logger/log.rb +236 -80
  68. data/lib/semantic_logger/loggable.rb +55 -95
  69. data/lib/semantic_logger/logger.rb +31 -5
  70. data/lib/semantic_logger/metric/new_relic.rb +32 -17
  71. data/lib/semantic_logger/metric/signalfx.rb +21 -12
  72. data/lib/semantic_logger/metric/statsd.rb +26 -8
  73. data/lib/semantic_logger/processor.rb +194 -5
  74. data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
  75. data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
  76. data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
  77. data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
  78. data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
  79. data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
  80. data/lib/semantic_logger/queue_processor/request.rb +48 -10
  81. data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
  82. data/lib/semantic_logger/queue_processor.rb +426 -176
  83. data/lib/semantic_logger/ractor_copy.rb +120 -0
  84. data/lib/semantic_logger/ractor_processor.rb +82 -0
  85. data/lib/semantic_logger/ractor_receiver.rb +282 -0
  86. data/lib/semantic_logger/ractor_support.rb +28 -0
  87. data/lib/semantic_logger/redactor.rb +310 -0
  88. data/lib/semantic_logger/semantic_logger.rb +313 -79
  89. data/lib/semantic_logger/subscriber.rb +169 -15
  90. data/lib/semantic_logger/sync_processor.rb +226 -26
  91. data/lib/semantic_logger/test/capture_log_events.rb +20 -2
  92. data/lib/semantic_logger/test/minitest.rb +2 -1
  93. data/lib/semantic_logger/test/rspec.rb +1 -1
  94. data/lib/semantic_logger/trap_safe.rb +113 -0
  95. data/lib/semantic_logger/utils.rb +266 -79
  96. data/lib/semantic_logger/version.rb +1 -1
  97. data/lib/semantic_logger.rb +35 -2
  98. metadata +22 -2
@@ -3,15 +3,32 @@ module SemanticLogger
3
3
  # Only output one line for each log entry.
4
4
  #
5
5
  # Notes:
6
- # * New lines and carriage returns are stripped from every text field: the name, thread name,
7
- # file name, tags, named tags, message, payload and exception message.
8
- # With `escape_control_chars: true` they, and all other control characters, are escaped
9
- # instead, like the other text formatters.
6
+ # * Line breaks are stripped from every text field: the name, thread name, file name, tags,
7
+ # named tags, message, payload and exception message. They are new lines and carriage
8
+ # returns, and the Unicode next line (U+0085), line separator (U+2028), and paragraph
9
+ # separator (U+2029), at which editors, browsers, and some log viewers also start a new line.
10
+ # With `escape_control_chars: true` they are escaped instead, the Unicode ones in the
11
+ # `\uXXXX` form, and so are the other control characters, as the other text formatters
12
+ # escape them: the ASCII ones, and the Unicode ones that control how text is shown, such as
13
+ # the bidirectional controls (see Base::UNICODE_CONTROL_CHARS).
10
14
  # * Exceptions only include the class and message, the stack trace is not shown.
11
15
  # * Exception details, when enabled, are joined onto the same line.
12
16
  class OneLine < Default
17
+ # The line breaks that Unicode adds to the line feed and carriage return: next line, line
18
+ # separator, and paragraph separator.
19
+ UNICODE_LINE_BREAKS = "\u0085\u2028\u2029".freeze
20
+
21
+ # Every line break, as a String#delete character set.
22
+ LINE_BREAKS = "\r\n#{UNICODE_LINE_BREAKS}".freeze
23
+
24
+ # A line break, with the white space around it, between two lines of exception details. A
25
+ # carriage return before a line feed is part of that white space.
26
+ DETAILS_LINE_BREAK = /\s*[\n#{UNICODE_LINE_BREAKS}]\s*/
27
+
28
+ # Shortened before its line breaks are removed or escaped, as the other text formatters do
29
+ # (see Default#shortened_thread_name), so that an escape is never cut in two.
13
30
  def thread_name
14
- format("%.30s", one_line(log.thread_name))
31
+ one_line(shortened_thread_name)
15
32
  end
16
33
 
17
34
  def tags
@@ -32,10 +49,18 @@ module SemanticLogger
32
49
  one_line(log.name) if log.name
33
50
  end
34
51
 
35
- # A log call can supply the backtrace that the file name comes from.
52
+ # A log call can supply the backtrace that the file name and method come from.
36
53
  def file_name_and_line
37
54
  file, line = log.file_name_and_line(true)
38
- "#{one_line(file)}:#{line}" if file
55
+ return unless file
56
+
57
+ label = method_name
58
+ label ? "#{one_line(file)}:#{line} #{label}" : "#{one_line(file)}:#{line}"
59
+ end
60
+
61
+ def method_name
62
+ label = log.call_site&.label
63
+ one_line(label) if label
39
64
  end
40
65
 
41
66
  def message
@@ -50,8 +75,9 @@ module SemanticLogger
50
75
  def exception
51
76
  return unless log.exception
52
77
 
78
+ message = one_line(Utils.exception_message(log.exception))
53
79
  details = exception_details
54
- "-- Exception: #{log.exception.class}: #{one_line(log.exception.message)}#{" -- #{details}" if details}"
80
+ "-- Exception: #{Utils.exception_name(log.exception)}: #{message}#{" -- #{details}" if details}"
55
81
  end
56
82
 
57
83
  # The exception details, with each line trimmed and joined by a space.
@@ -59,7 +85,7 @@ module SemanticLogger
59
85
  details = exception_details_for(log.exception) if log.exception
60
86
  return unless details
61
87
 
62
- one_line(Utils.encode_utf8_string(details).split(/\s*\n\s*/).reject(&:empty?).join(" ").strip)
88
+ one_line(Utils.encode_utf8_string(details).split(DETAILS_LINE_BREAK).reject(&:empty?).join(" ").strip)
63
89
  end
64
90
 
65
91
  private
@@ -70,12 +96,20 @@ module SemanticLogger
70
96
  end
71
97
 
72
98
  # Returns [String] the value without line breaks, so that the entry stays on one line.
99
+ # With `escape_control_chars: true`, #escape_control_characters has already escaped them, the
100
+ # Unicode ones as in "\u2028".
73
101
  #
74
102
  # A String with invalid bytes, such as a request parameter decoded from "%FF", or in another
75
103
  # encoding, such as a binary request body, is converted to valid UTF-8 first, since deleting
76
104
  # from it, or joining it with UTF-8 text, would raise and lose the entry.
77
105
  def one_line(value)
78
- Utils.encode_utf8_string(escape_control_characters(value.to_s)).delete("\r\n")
106
+ text = Utils.encode_utf8_string(escape_control_characters(value.to_s))
107
+ # #escape_control_characters has already escaped every line break.
108
+ return text if escape_control_chars
109
+ # Most text is ASCII, and deleting two characters is quicker than deleting five.
110
+ return text.delete("\r\n") if text.ascii_only?
111
+
112
+ text.delete(LINE_BREAKS)
79
113
  end
80
114
  end
81
115
  end
@@ -1,57 +1,214 @@
1
- require "json"
2
1
  module SemanticLogger
3
2
  module Formatters
4
- class OpenTelemetry < Raw
5
- # primitives allowed by OTLP logs in Ruby: String, Integer, Float, TrueClass, FalseClass
6
- PRIMS = [String, Integer, Float, TrueClass, FalseClass].freeze
3
+ # Maps a log entry to an OpenTelemetry log record.
4
+ #
5
+ # Returns the keyword arguments for the OpenTelemetry Logs API's `Logger#on_emit`, other than
6
+ # the context, which the appender captures:
7
+ # severity_text: the level, such as "info"
8
+ # severity_number: the OpenTelemetry severity number of the level
9
+ # timestamp: the time the entry was logged
10
+ # body: the message, or nil when the entry has none
11
+ # attributes: the named tags and the payload, then the entry's own fields
12
+ #
13
+ # Each named tag and payload key is an attribute of its own, the payload taking precedence
14
+ # over a named tag of the same name. The entry's own fields use the attribute names of the
15
+ # OpenTelemetry semantic conventions where there is one, and are otherwise prefixed with
16
+ # "semantic_logger.", so that they do not collide with the application's keys:
17
+ # exception.type, exception.message, exception.stacktrace (with each exception that caused it)
18
+ # code.file.path, code.line.number (where the entry was logged, when it has a call site or
19
+ # captured a backtrace), code.function.name (when it has a call site)
20
+ # thread.name, process.pid
21
+ # semantic_logger.name, semantic_logger.tags, semantic_logger.duration_ms,
22
+ # semantic_logger.metric, semantic_logger.metric_amount
23
+ # semantic_logger.dimensions.<key>, one for each dimension of a metric
24
+ # A named tag or payload key named like one of these is prefixed with its source, as in
25
+ # "payload.exception.type", so that logged data cannot replace or forge the entry's own fields.
26
+ #
27
+ # The host, application, and environment are not attributes: they describe the process, which
28
+ # OpenTelemetry reports in the resource of every export. The process id is, since the resource
29
+ # holds the id of the process that configured OpenTelemetry, not of a forked child.
30
+ #
31
+ # An attribute value is a String, Integer, Float, true, or false, or an Array of them of one
32
+ # type, since the OpenTelemetry SDK reports any other value as an error, which, when that error
33
+ # is logged to this appender with the same named tags, repeats until the stack overflows. See
34
+ # #coerce_value.
35
+ class OpenTelemetry < Base
36
+ # The entry's own attributes.
37
+ EXCEPTION_TYPE = "exception.type".freeze
38
+ EXCEPTION_MESSAGE = "exception.message".freeze
39
+ EXCEPTION_STACKTRACE = "exception.stacktrace".freeze
40
+ CODE_FILE_PATH = "code.file.path".freeze
41
+ CODE_LINE_NUMBER = "code.line.number".freeze
42
+ CODE_FUNCTION_NAME = "code.function.name".freeze
43
+ THREAD_NAME = "thread.name".freeze
44
+ PROCESS_PID = "process.pid".freeze
45
+ NAME = "semantic_logger.name".freeze
46
+ TAGS = "semantic_logger.tags".freeze
47
+ DURATION_MS = "semantic_logger.duration_ms".freeze
48
+ METRIC = "semantic_logger.metric".freeze
49
+ METRIC_AMOUNT = "semantic_logger.metric_amount".freeze
50
+ DIMENSIONS_PREFIX = "semantic_logger.dimensions.".freeze
7
51
 
8
- # Returns the attributes hash submitted to the OpenTelemetry SDK.
52
+ ENTRY_ATTRIBUTES = [
53
+ EXCEPTION_TYPE, EXCEPTION_MESSAGE, EXCEPTION_STACKTRACE, CODE_FILE_PATH, CODE_LINE_NUMBER,
54
+ CODE_FUNCTION_NAME, THREAD_NAME, PROCESS_PID, NAME, TAGS, DURATION_MS, METRIC, METRIC_AMOUNT
55
+ ].to_h { |name| [name, true] }.freeze
56
+
57
+ # Prefixes for a named tag or payload key named like one of the entry's own attributes.
58
+ NAMED_TAGS_PREFIX = "named_tags.".freeze
59
+ PAYLOAD_PREFIX = "payload.".freeze
60
+
61
+ # Returns [Hash] the keyword arguments for `Logger#on_emit`.
9
62
  #
10
63
  # Unlike the JSON formatters there is no single `.to_json` boundary here:
11
- # the hash is handed to the OTLP exporter, which requires valid UTF-8. Cleanse
64
+ # the record is handed to the OTLP exporter, which requires valid UTF-8. Cleanse
12
65
  # the whole structure so binary / non UTF-8 strings cannot break the export.
13
66
  def call(log, logger)
14
- Utils.encode_utf8(super)
15
- end
67
+ self.log = log
68
+ self.logger = logger
16
69
 
17
- # Log level
18
- def level
19
- hash[:level] = log.level.to_s
20
- hash[:level_index] = severity_number(log.level)
70
+ Utils.encode_utf8(
71
+ {
72
+ severity_text: log.level.to_s,
73
+ severity_number: severity_number(log.level),
74
+ timestamp: log.time,
75
+ body: body,
76
+ attributes: attributes
77
+ }
78
+ )
21
79
  end
22
80
 
23
- # Payload is submitted directly as attributes, which must be a Hash. A log call can supply
24
- # any value as the payload, such as a String parsed from a request body, so any other value
25
- # is submitted as a single "payload" attribute.
26
- def payload
27
- return unless log.payload?
81
+ # Returns [String] the message, or nil when the entry has none, such as a metric only entry,
82
+ # or an exception logged without a message.
83
+ def body
84
+ log.cleansed_message if log.message
85
+ end
28
86
 
29
- hash[:payload] = coerce_map(log.payload.is_a?(Hash) ? log.payload : {payload: log.payload})
87
+ # Returns [Hash] the named tags and the payload, then the entry's own fields.
88
+ #
89
+ # A payload that is not a Hash, such as a String parsed from a request body, is submitted as a
90
+ # single "payload" attribute.
91
+ def attributes
92
+ attributes = {}
93
+ add_logged(attributes, NAMED_TAGS_PREFIX, log.named_tags) if log.named_tags.is_a?(Hash)
94
+ if log.payload?
95
+ add_logged(attributes, PAYLOAD_PREFIX, log.payload.is_a?(Hash) ? log.payload : {payload: log.payload})
96
+ end
97
+ add_entry(attributes)
98
+ add_exception(attributes)
99
+ attributes
30
100
  end
31
101
 
32
102
  private
33
103
 
34
- # An attribute value: a primitive, or an Array of them.
104
+ # Adds each key and value of a named tags or payload Hash, prefixing a key named like one of
105
+ # the entry's own attributes with its source.
106
+ def add_logged(attributes, prefix, hash)
107
+ hash.each_pair do |key, value|
108
+ name = Utils.encode_utf8_string(key.to_s)
109
+ name = prefix + name if ENTRY_ATTRIBUTES.key?(name) || name.start_with?(DIMENSIONS_PREFIX)
110
+ add(attributes, name, value)
111
+ end
112
+ end
113
+
114
+ def add_entry(attributes)
115
+ add(attributes, NAME, log.name)
116
+ add(attributes, THREAD_NAME, log.thread_name)
117
+ add(attributes, PROCESS_PID, pid)
118
+
119
+ add_code(attributes)
120
+
121
+ add(attributes, TAGS, log.tags) if log.tags && !log.tags.empty?
122
+ add(attributes, DURATION_MS, log.duration)
123
+ add(attributes, METRIC, log.metric)
124
+ add(attributes, METRIC_AMOUNT, log.metric_amount)
125
+ add_dimensions(attributes)
126
+ end
127
+
128
+ # Each dimension of a metric is an attribute of its own, so that it can be queried.
129
+ def add_dimensions(attributes)
130
+ return unless log.dimensions?
131
+
132
+ log.dimensions.each_pair do |key, value|
133
+ add(attributes, DIMENSIONS_PREFIX + Utils.encode_utf8_string(key.to_s), value)
134
+ end
135
+ end
136
+
137
+ # Where the entry was logged: its call site, or else the first line of the backtrace it captured.
138
+ # Not the logged exception's backtrace, whose location is in its stacktrace.
139
+ def add_code(attributes)
140
+ call_site = log.call_site
141
+ file, line = call_site ? call_site.file_name_and_line : log.extract_file_and_line(log.backtrace)
142
+ return unless file
143
+
144
+ add(attributes, CODE_FILE_PATH, file)
145
+ add(attributes, CODE_LINE_NUMBER, line)
146
+ add(attributes, CODE_FUNCTION_NAME, call_site&.label)
147
+ end
148
+
149
+ # The logged exception, with the backtrace of each exception that caused it in its stacktrace.
150
+ # With `log_exception_details`, the message is followed by the detail Ruby adds to it, on a
151
+ # line of its own, as in Exception#detailed_message.
152
+ #
153
+ # The type is the class as Utils.exception_name names it, as every formatter does: as Ruby names
154
+ # it, without asking the class, such as `#<Class:0x000...>` for an anonymous class. The
155
+ # message is read with Utils.exception_message, so that a #message that raises, or does not
156
+ # return a String, cannot lose the entry.
157
+ def add_exception(attributes)
158
+ exception = log.exception
159
+ return unless exception
160
+
161
+ add(attributes, EXCEPTION_TYPE, Utils.exception_name(exception))
162
+ message = Utils.exception_message(exception)
163
+ details = exception_details_for(exception)
164
+ add(attributes, EXCEPTION_MESSAGE, details ? "#{message}\n#{details}" : message)
165
+ stacktrace = exception_backtrace
166
+ add(attributes, EXCEPTION_STACKTRACE, stacktrace) unless stacktrace.empty?
167
+ end
168
+
169
+ # Adds the value as an attribute, unless it is nil.
170
+ def add(attributes, name, value)
171
+ attributes[name] = coerce_value(value) unless value.nil?
172
+ end
173
+
174
+ # An attribute value: a primitive, or an Array of primitives of one type. The elements of an
175
+ # Array that mixes types, such as [1, "two"], are all written as Strings.
35
176
  def coerce_value(v)
36
- v.is_a?(Array) ? v.map { |e| coerce_element(e) }.compact : coerce_element(v)
177
+ return coerce_element(v) unless v.is_a?(Array)
178
+
179
+ values = v.map { |e| coerce_element(e) }.compact
180
+ one_type?(values) ? values : values.map(&:to_s)
181
+ end
182
+
183
+ # Whether the coerced values are all Strings, all booleans, or all numbers.
184
+ def one_type?(values)
185
+ case values.first
186
+ when String
187
+ values.all?(String)
188
+ when true, false
189
+ values.all? { |e| [true, false].include?(e) }
190
+ else
191
+ values.all?(Numeric)
192
+ end
37
193
  end
38
194
 
39
195
  # A primitive as-is, and anything else as a String: a structure (such as a Hash, Struct, or
40
196
  # Set) as JSON, depth-limited and repaired by Utils.to_json, rather than its #to_s, which is
41
- # its #inspect and would expose the instance variables of any object it holds.
197
+ # its #inspect and would expose the instance variables of any object it holds. A subclass of
198
+ # String, such as ActiveSupport::SafeBuffer, is copied to a String, which is all the SDK accepts.
42
199
  def coerce_element(v)
43
200
  case v
44
- when *PRIMS, NilClass then v
45
- else Utils.structure?(v) ? Utils.to_json(v) : v.to_s
201
+ when Integer, Float, true, false, nil
202
+ v
203
+ when String
204
+ plain_string(v)
205
+ else
206
+ plain_string(Utils.structure?(v) ? Utils.to_json(v) : v.to_s)
46
207
  end
47
208
  end
48
209
 
49
- def coerce_map(h)
50
- h.each_with_object({}) do |(k, v), out|
51
- next if v.nil?
52
-
53
- out[k.to_s] = coerce_value(v)
54
- end
210
+ def plain_string(string)
211
+ string.instance_of?(String) ? string : String.new(string)
55
212
  end
56
213
 
57
214
  def severity_number(severity)
@@ -39,6 +39,8 @@ module SemanticLogger
39
39
  # pid Process id.
40
40
  # file_name Ruby file name that logged the message, e.g. "app.rb".
41
41
  # line Line number within the Ruby file, e.g. 42.
42
+ # method Method that logged the message, e.g. "Order#submit", when
43
+ # the entry has a call site (see SemanticLogger.call_site_level).
42
44
  # tags Tags, comma separated.
43
45
  # named_tags All named tags, or one tag with named_tags:key.
44
46
  # host Host name.
@@ -69,6 +71,7 @@ module SemanticLogger
69
71
  pid: false,
70
72
  file_name: false,
71
73
  line: false,
74
+ method: false,
72
75
  tags: false,
73
76
  named_tags: true,
74
77
  host: false,
@@ -76,6 +79,11 @@ module SemanticLogger
76
79
  environment: false
77
80
  }.freeze
78
81
 
82
+ # The formatter method of each directive not named like it: a method named `method` would
83
+ # replace Object#method.
84
+ DIRECTIVE_METHODS = {method: :method_name}.freeze
85
+ private_constant :DIRECTIVE_METHODS
86
+
79
87
  # A single interpolated directive within a compiled pattern.
80
88
  Token = Struct.new(:method_name, :arguments)
81
89
  private_constant :Token
@@ -92,7 +100,7 @@ module SemanticLogger
92
100
  # Parse the pattern once, up front, so that formatting every log entry
93
101
  # is just a walk over the pre-compiled tokens (no regex on the hot path).
94
102
  # Unknown directives raise here, at configuration time, not per log.
95
- @tokens = compile(pattern)
103
+ @tokens = compile(compilable(pattern)).map { |token| joinable_token(token) }
96
104
  end
97
105
 
98
106
  # Formatted timestamp. With a strftime format argument, e.g.
@@ -126,12 +134,12 @@ module SemanticLogger
126
134
 
127
135
  # Class of the logged exception, e.g. "RuntimeError".
128
136
  def exception_class
129
- log.exception&.class
137
+ Utils.exception_name(log.exception) if log.exception
130
138
  end
131
139
 
132
140
  # Message of the logged exception.
133
141
  def exception_message
134
- escape_control_characters(log.exception&.message)
142
+ escape_control_characters(Utils.exception_message(log.exception)) if log.exception
135
143
  end
136
144
 
137
145
  # Backtrace of the logged exception.
@@ -182,19 +190,23 @@ module SemanticLogger
182
190
  end
183
191
  end
184
192
 
193
+ # The host, application, and environment names are converted as the formatter's own text is
194
+ # (see Base#joinable_text), since they can be in another encoding, such as when read from the
195
+ # environment where no locale is set.
196
+
185
197
  # Host name.
186
198
  def host
187
- logger&.host if log_host
199
+ joinable_text(logger&.host) if log_host
188
200
  end
189
201
 
190
202
  # Application name.
191
203
  def application
192
- logger&.application if log_application
204
+ joinable_text(logger&.application) if log_application
193
205
  end
194
206
 
195
207
  # Environment name.
196
208
  def environment
197
- logger&.environment if log_environment
209
+ joinable_text(logger&.environment) if log_environment
198
210
  end
199
211
 
200
212
  def call(log, logger)
@@ -208,6 +220,46 @@ module SemanticLogger
208
220
 
209
221
  private
210
222
 
223
+ # Returns [String] the pattern to compile, which raises when the pattern is not valid text, as
224
+ # one that is not valid UTF-8 always has, rather than writing a different pattern. Binary is
225
+ # the exception, since a binary pattern has always been accepted.
226
+ #
227
+ # * UTF-8, or another ASCII-compatible encoding that says which characters its bytes are, such
228
+ # as ISO-8859-1: as it is, so that a %{named_tags:...} key in it still names the
229
+ # application's own key in that encoding (see #joinable_token).
230
+ # * US-ASCII with bytes outside ASCII, such as a pattern read from a file or the environment
231
+ # where no locale is set: read as the UTF-8 it usually holds, as the fields are.
232
+ # * Binary: read as UTF-8 too, with any byte that is not UTF-8 replaced by U+FFFD, so that the
233
+ # pattern keeps its shape: a dropped byte could join a "%" to a "{".
234
+ # * Any other encoding, such as UTF-16: converted to UTF-8.
235
+ def compilable(pattern)
236
+ encoding = pattern.encoding
237
+ return pattern if encoding == Encoding::UTF_8 || pattern.ascii_only?
238
+ return String.new(pattern, encoding: Encoding::UTF_8).scrub if encoding == Encoding::BINARY
239
+ return String.new(pattern, encoding: Encoding::UTF_8) if encoding == Encoding::US_ASCII
240
+
241
+ encoding.ascii_compatible? ? pattern : pattern.encode(Encoding::UTF_8)
242
+ end
243
+
244
+ # Returns the compiled token with its text in UTF-8, so that a pattern with characters outside
245
+ # ASCII in, say, ISO-8859-1 can be joined with the entry's fields: a literal, or the format of
246
+ # %{time:...}. The key of %{named_tags:...} is left as written, so that it still names the
247
+ # application's own key.
248
+ #
249
+ # Converted strictly, since this runs once, when the formatter is built: a character with no
250
+ # UTF-8 equivalent, or an encoding that cannot be converted, raises rather than being dropped.
251
+ def joinable_token(token)
252
+ return utf8_pattern_text(token) if token.is_a?(String)
253
+ return token unless token.method_name == :time
254
+
255
+ Token.new(token.method_name, token.arguments.map { |format| utf8_pattern_text(format) }).freeze
256
+ end
257
+
258
+ # Returns [String] the pattern's own text in UTF-8, frozen (see #joinable_token).
259
+ def utf8_pattern_text(text)
260
+ (text.encoding == Encoding::UTF_8 || text.ascii_only? ? text : text.encode(Encoding::UTF_8)).freeze
261
+ end
262
+
211
263
  # Parse the pattern string into an array of tokens: frozen literal strings
212
264
  # and Token structs for each %{directive} placeholder. %%{...} is an escape
213
265
  # that produces a literal %{...}.
@@ -222,12 +274,12 @@ module SemanticLogger
222
274
  if match.start_with?("%%")
223
275
  tokens << match[1..].freeze
224
276
  else
225
- name, arg = match[/\{([^}]+)\}/, 1].split(":", 2)
277
+ name, arg = match[2..-2].split(":", 2)
226
278
  name = name.strip.to_sym
227
279
  raise(ArgumentError, "Invalid pattern directive: %{#{name}}") unless DIRECTIVES.key?(name)
228
280
  raise(ArgumentError, "%{#{name}} does not accept an argument") if arg && !DIRECTIVES[name]
229
281
 
230
- tokens << Token.new(name, arg ? [arg.strip] : []).freeze
282
+ tokens << Token.new(DIRECTIVE_METHODS.fetch(name, name), arg ? [arg.strip] : []).freeze
231
283
  end
232
284
 
233
285
  pos = current.end(0)
@@ -47,13 +47,16 @@ module SemanticLogger
47
47
  hash[:thread] = log.thread_name
48
48
  end
49
49
 
50
- # Ruby file name and line number that logged the message.
50
+ # Ruby file name and line number that logged the message, and the method when the entry has a
51
+ # call site (see SemanticLogger.call_site_level).
51
52
  def file_name_and_line
52
53
  file, line = log.file_name_and_line
53
54
  return unless file
54
55
 
55
- hash[:file] = file
56
- hash[:line] = line.to_i
56
+ hash[:file] = file
57
+ hash[:line] = line.to_i
58
+ label = log.call_site&.label
59
+ hash[:method] = label if label
57
60
  end
58
61
 
59
62
  # Tags
@@ -89,25 +92,33 @@ module SemanticLogger
89
92
  hash[:payload] = log.payload if log.payload.respond_to?(:empty?) && !log.payload.empty?
90
93
  end
91
94
 
92
- # Exception
95
+ # Exception, with the stack trace of each exception cleaned by the backtrace cleaner (see
96
+ # Base#initialize).
97
+ #
98
+ # Each exception is named as Utils.exception_name names it, as every formatter does: as Ruby
99
+ # names its class, without asking the class, such as `#<Class:0x000...>` for an anonymous class,
100
+ # which has no name. Its message is read with Utils.exception_message, so that a #message
101
+ # that raises, or does not return a String, cannot lose the entry.
93
102
  def exception
94
103
  return unless log.exception
95
104
 
96
- root = hash
105
+ cleaner = active_backtrace_cleaner
106
+ root = hash
97
107
  log.each_exception do |exception, i|
98
108
  name = i.zero? ? :exception : :cause
99
- root[name] = {name: exception.class.name, message: exception.message}
109
+ root[name] = {name: Utils.exception_name(exception), message: Utils.exception_message(exception)}
100
110
  details = exception_details_for(exception)
101
111
  root[name][:details] = details if details
102
- root[name][:stack_trace] = exception.backtrace
112
+ root[name][:stack_trace] = stack_trace_of(exception, cleaner)
103
113
  root = root[name]
104
114
  end
105
115
  end
106
116
 
107
- # Metric
117
+ # Metric, its amount, and its dimensions
108
118
  def metric
109
119
  hash[:metric] = log.metric if log.metric
110
120
  hash[:metric_amount] = log.metric_amount if log.metric_amount
121
+ hash[:dimensions] = log.dimensions if log.dimensions?
111
122
  end
112
123
 
113
124
  # Returns log messages in Hash format
@@ -135,6 +146,25 @@ module SemanticLogger
135
146
 
136
147
  hash
137
148
  end
149
+
150
+ private
151
+
152
+ # Returns [Array<String>] the backtrace to write for the exception.
153
+ #
154
+ # A cleaned backtrace is frozen and shared with the other appenders (see Log#backtrace_of),
155
+ # so the Hash gets a copy that a subclass can change, as it could change the exception's own
156
+ # backtrace before cleaning.
157
+ def stack_trace_of(exception, cleaner)
158
+ backtrace = log.backtrace_of(exception, cleaner: cleaner)
159
+ cleaner && backtrace ? backtrace.map(&:dup) : backtrace
160
+ end
161
+
162
+ # Returns [String] the lines of a stack trace, such as the Hash's `stack_trace`, as one text, one
163
+ # per line, for a subclass that writes it as text, joined as Log#backtrace_to_s joins its lines
164
+ # (see Utils.join_lines). Lines that cannot be joined, such as a UTF-16 line, or a binary line
165
+ # next to a UTF-8 one, in a backtrace that the application set, are converted to valid UTF-8,
166
+ # rather than losing the entry.
167
+ def stack_trace(lines) = Utils.join_lines(lines)
138
168
  end
139
169
  end
140
170
  end