semantic_logger 4.18.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +51 -81
  3. data/Rakefile +39 -3
  4. data/docs/api.md +480 -0
  5. data/docs/appenders.md +990 -0
  6. data/docs/config.md +605 -0
  7. data/docs/index.md +197 -0
  8. data/docs/log.md +108 -0
  9. data/docs/metrics.md +168 -0
  10. data/docs/operations.md +400 -0
  11. data/docs/rails.md +816 -0
  12. data/docs/security.md +119 -0
  13. data/docs/testing.md +285 -0
  14. data/docs/upgrading.md +409 -0
  15. data/lib/semantic_logger/appender/async.rb +86 -173
  16. data/lib/semantic_logger/appender/cloudwatch_logs.rb +8 -6
  17. data/lib/semantic_logger/appender/elasticsearch.rb +8 -187
  18. data/lib/semantic_logger/appender/elasticsearch_base.rb +212 -0
  19. data/lib/semantic_logger/appender/elasticsearch_http.rb +16 -9
  20. data/lib/semantic_logger/appender/file.rb +25 -4
  21. data/lib/semantic_logger/appender/graylog.rb +2 -2
  22. data/lib/semantic_logger/appender/http.rb +27 -2
  23. data/lib/semantic_logger/appender/io.rb +8 -4
  24. data/lib/semantic_logger/appender/kafka.rb +2 -2
  25. data/lib/semantic_logger/appender/loki.rb +2 -4
  26. data/lib/semantic_logger/appender/mongodb.rb +3 -6
  27. data/lib/semantic_logger/appender/new_relic.rb +6 -0
  28. data/lib/semantic_logger/appender/new_relic_logs.rb +4 -3
  29. data/lib/semantic_logger/appender/open_telemetry.rb +10 -7
  30. data/lib/semantic_logger/appender/opensearch.rb +36 -0
  31. data/lib/semantic_logger/appender/rabbitmq.rb +5 -4
  32. data/lib/semantic_logger/appender/sentry.rb +5 -0
  33. data/lib/semantic_logger/appender/sentry_ruby.rb +2 -2
  34. data/lib/semantic_logger/appender/splunk.rb +2 -2
  35. data/lib/semantic_logger/appender/splunk_http.rb +3 -3
  36. data/lib/semantic_logger/appender/syslog.rb +11 -7
  37. data/lib/semantic_logger/appender/tcp.rb +4 -3
  38. data/lib/semantic_logger/appender/udp.rb +2 -2
  39. data/lib/semantic_logger/appender/wrapper.rb +5 -4
  40. data/lib/semantic_logger/appender.rb +36 -22
  41. data/lib/semantic_logger/appenders.rb +27 -5
  42. data/lib/semantic_logger/base.rb +100 -23
  43. data/lib/semantic_logger/concerns/compatibility.rb +2 -2
  44. data/lib/semantic_logger/core_ext/process.rb +34 -0
  45. data/lib/semantic_logger/formatters/base.rb +46 -7
  46. data/lib/semantic_logger/formatters/color.rb +9 -3
  47. data/lib/semantic_logger/formatters/default.rb +11 -4
  48. data/lib/semantic_logger/formatters/ecs.rb +151 -0
  49. data/lib/semantic_logger/formatters/fluentd.rb +15 -4
  50. data/lib/semantic_logger/formatters/json.rb +6 -1
  51. data/lib/semantic_logger/formatters/logfmt.rb +6 -4
  52. data/lib/semantic_logger/formatters/loki.rb +4 -4
  53. data/lib/semantic_logger/formatters/open_telemetry.rb +15 -5
  54. data/lib/semantic_logger/formatters/pattern.rb +237 -0
  55. data/lib/semantic_logger/formatters/raw.rb +2 -2
  56. data/lib/semantic_logger/formatters/signalfx.rb +2 -2
  57. data/lib/semantic_logger/formatters/syslog.rb +16 -4
  58. data/lib/semantic_logger/formatters/syslog_cee.rb +3 -2
  59. data/lib/semantic_logger/formatters.rb +2 -0
  60. data/lib/semantic_logger/log.rb +18 -4
  61. data/lib/semantic_logger/logger.rb +2 -2
  62. data/lib/semantic_logger/metric/new_relic.rb +2 -2
  63. data/lib/semantic_logger/metric/signalfx.rb +6 -4
  64. data/lib/semantic_logger/metric/statsd.rb +2 -2
  65. data/lib/semantic_logger/processor.rb +21 -0
  66. data/lib/semantic_logger/queue_processor.rb +372 -0
  67. data/lib/semantic_logger/reporters/minitest.rb +8 -8
  68. data/lib/semantic_logger/semantic_logger.rb +103 -11
  69. data/lib/semantic_logger/subscriber.rb +15 -2
  70. data/lib/semantic_logger/sync_processor.rb +25 -3
  71. data/lib/semantic_logger/test/capture_log_events.rb +3 -3
  72. data/lib/semantic_logger/test/minitest.rb +8 -4
  73. data/lib/semantic_logger/test/rspec.rb +249 -0
  74. data/lib/semantic_logger/utils.rb +83 -4
  75. data/lib/semantic_logger/version.rb +1 -1
  76. data/lib/semantic_logger.rb +9 -0
  77. metadata +28 -8
  78. data/lib/semantic_logger/appender/async_batch.rb +0 -93
@@ -37,13 +37,13 @@ module SemanticLogger
37
37
  #
38
38
  # logger = SemanticLogger['test']
39
39
  # logger.info 'Hello World'
40
- def initialize(io, **args, &block)
40
+ def initialize(io, **args, &)
41
41
  @io = io
42
42
  unless @io.respond_to?(:write)
43
43
  raise(ArgumentError, "SemanticLogging::Appender::IO io is not a valid IO instance: #{io.inspect}")
44
44
  end
45
45
 
46
- super(**args, &block)
46
+ super(**args, &)
47
47
  end
48
48
 
49
49
  def log(log)
@@ -60,8 +60,12 @@ module SemanticLogger
60
60
  @io.flush if @io.respond_to?(:flush)
61
61
  end
62
62
 
63
- def console_output?
64
- [$stderr, $stdout].include?(@io)
63
+ def console_stream
64
+ if @io.equal?($stdout)
65
+ :stdout
66
+ elsif @io.equal?($stderr)
67
+ :stderr
68
+ end
65
69
  end
66
70
  end
67
71
  end
@@ -127,7 +127,7 @@ module SemanticLogger
127
127
  ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
128
128
  topic: "log_messages", partition: nil, partition_key: nil, key: nil,
129
129
  delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
130
- metrics: true, **args, &block)
130
+ metrics: true, **args, &)
131
131
  @seed_brokers = seed_brokers
132
132
  @client_id = client_id
133
133
  @connect_timeout = connect_timeout
@@ -144,7 +144,7 @@ module SemanticLogger
144
144
  @delivery_interval = delivery_interval
145
145
  @required_acks = required_acks
146
146
 
147
- super(metrics: metrics, **args, &block)
147
+ super(metrics: metrics, **args, &)
148
148
  reopen
149
149
  end
150
150
 
@@ -39,14 +39,12 @@ module SemanticLogger
39
39
  header: {"Content-Type" => "application/json"},
40
40
  path: INGESTION_PATH,
41
41
  **args,
42
- &block)
43
-
44
- super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &block)
42
+ &)
43
+ super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &)
45
44
  end
46
45
 
47
46
  def log(log)
48
47
  message = formatter.call(log, self)
49
- puts message
50
48
  logger.trace(message)
51
49
  post(message)
52
50
  end
@@ -36,11 +36,8 @@ module SemanticLogger
36
36
  # Example:
37
37
  # require 'semantic_logger'
38
38
  #
39
- # client = Mongo::MongoClient.new
40
- # database = client['test']
41
- #
42
39
  # appender = SemanticLogger::Appender::MongoDB.new(
43
- # db: database,
40
+ # uri: 'mongodb://127.0.0.1:27017/test',
44
41
  # collection_size: 1024**3 # 1.gigabyte
45
42
  # )
46
43
  # SemanticLogger.add_appender(appender: appender)
@@ -110,7 +107,7 @@ module SemanticLogger
110
107
  collection_size: 1024**3,
111
108
  collection_max: nil,
112
109
  **args,
113
- &block)
110
+ &)
114
111
  @client = Mongo::Client.new(uri, logger: logger)
115
112
  @collection_name = collection_name
116
113
  @options = {
@@ -125,7 +122,7 @@ module SemanticLogger
125
122
  # Create the collection and necessary indexes
126
123
  create_indexes
127
124
 
128
- super(**args, &block)
125
+ super(**args, &)
129
126
  end
130
127
 
131
128
  # After forking an active process call #reopen to re-open
@@ -33,6 +33,12 @@ module SemanticLogger
33
33
  # Proc: Only include log messages where the supplied Proc returns true
34
34
  # The Proc must return true or false.
35
35
  def initialize(level: :error, **args, &block)
36
+ Kernel.warn(
37
+ "SemanticLogger::Appender::NewRelic (appender: :new_relic) is deprecated and will be removed in v6. " \
38
+ "It reports errors via newrelic_rpm and is superseded by the log-forwarding appender " \
39
+ "appender: :new_relic_logs.",
40
+ category: :deprecated
41
+ )
36
42
  super
37
43
  end
38
44
 
@@ -48,7 +48,8 @@ module SemanticLogger
48
48
  def initialize(formatter: SemanticLogger::Formatters::NewRelicLogs.new, **args, &block)
49
49
  super
50
50
 
51
- # Record NewRelic's "trace.id"/"entity.name"/"hostname"/etc, so we can include them later in the formatted output.
51
+ # Record NewRelic's "trace.id"/"entity.name"/"hostname"/etc, so we can include them
52
+ # later in the formatted output.
52
53
  # These are thread-local, so need to be captured as soon as the log-message is created.
53
54
  # https://rubydoc.info/gems/newrelic_rpm/NewRelic/Agent#linking_metadata-instance_method
54
55
  SemanticLogger.on_log(CAPTURE_CONTEXT)
@@ -58,8 +59,8 @@ module SemanticLogger
58
59
  def log(log)
59
60
  begin
60
61
  message = formatter.call(log, self) # Generate the structured log
61
- json_message = message.to_json # Convert the log to JSON
62
- level = log.level.to_s.upcase # Determine the log level
62
+ json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
63
+ level = log.level.to_s.upcase # Determine the log level
63
64
  self.class.log_newrelic(json_message, level)
64
65
  rescue JSON::GeneratorError => e
65
66
  warn("Failed to serialize log message to JSON: #{e.message}")
@@ -2,7 +2,8 @@ begin
2
2
  require "opentelemetry/logs"
3
3
  rescue LoadError
4
4
  raise LoadError,
5
- 'Gem opentelemetry-logs-sdk is required for logging to Open Telemetry. Please add the gem "opentelemetry-logs-sdk" to your Gemfile.'
5
+ "Gem opentelemetry-logs-sdk is required for logging to Open Telemetry. " \
6
+ 'Please add the gem "opentelemetry-logs-sdk" to your Gemfile.'
6
7
  end
7
8
 
8
9
  # Open Telemetry Appender
@@ -29,7 +30,11 @@ module SemanticLogger
29
30
  formatter: SemanticLogger::Formatters::OpenTelemetry.new,
30
31
  metrics: true,
31
32
  **args,
32
- &block)
33
+ &)
34
+ # Base#initialize (via Subscriber) overwrites @name with the class name,
35
+ # so call super first and then assign our own attributes.
36
+ super(formatter: formatter, metrics: metrics, **args, &)
37
+
33
38
  @name = name
34
39
  @version = version
35
40
  @provider = ::OpenTelemetry.logger_provider
@@ -38,8 +43,6 @@ module SemanticLogger
38
43
  # Capture the current Open Telemetry context when a log entry is captured.
39
44
  # Prevents duplicate subscribers as long as it is from a constant.
40
45
  SemanticLogger.on_log(CAPTURE_CONTEXT)
41
-
42
- super(formatter: formatter, metrics: metrics, **args, &block)
43
46
  end
44
47
 
45
48
  def log(log)
@@ -57,7 +60,7 @@ module SemanticLogger
57
60
  timestamp: time,
58
61
  body: body.transform_keys!(&:to_s),
59
62
  attributes: payload,
60
- context: log.context[:open_telemetry] || ::OpenTelemetry::Context.current
63
+ context: (log.context && log.context[:open_telemetry]) || ::OpenTelemetry::Context.current
61
64
  )
62
65
  true
63
66
  end
@@ -69,7 +72,7 @@ module SemanticLogger
69
72
  @provider.force_flush if @provider.respond_to?(:force_flush)
70
73
  rescue StandardError => e
71
74
  # Swallow to avoid noisy shutdown exceptions.
72
- SemanticLogger.logger.warn("Flush failed: #{e.class}: #{e.message}")
75
+ SemanticLogger::Processor.logger.warn("Flush failed: #{e.class}: #{e.message}")
73
76
  end
74
77
 
75
78
  # Close the appender and release resources.
@@ -78,7 +81,7 @@ module SemanticLogger
78
81
 
79
82
  @provider.shutdown if @provider.respond_to?(:shutdown)
80
83
  rescue StandardError => e
81
- SemanticLogger.logger.warn("Shutdown failed: #{e.class}: #{e.message}")
84
+ SemanticLogger::Processor.logger.warn("Shutdown failed: #{e.class}: #{e.message}")
82
85
  ensure
83
86
  @provider = nil
84
87
  end
@@ -0,0 +1,36 @@
1
+ begin
2
+ require "opensearch-ruby"
3
+ rescue LoadError
4
+ raise LoadError,
5
+ "Gem opensearch-ruby is required for logging to OpenSearch. " \
6
+ 'Please add the gem "opensearch-ruby" to your Gemfile.'
7
+ end
8
+
9
+ require "semantic_logger/appender/elasticsearch_base"
10
+
11
+ # Forward all log messages to OpenSearch (e.g. AWS OpenSearch).
12
+ #
13
+ # OpenSearch is a fork of Elasticsearch 7.10. The client API and bulk
14
+ # protocol are identical, so this appender reuses the same pipeline as the
15
+ # Elasticsearch appender; only the backing client gem differs. Use this
16
+ # appender (with the `opensearch-ruby` gem) instead of `:elasticsearch`
17
+ # when talking to an OpenSearch server, since modern `elasticsearch` gems
18
+ # reject non-Elasticsearch servers via a product check.
19
+ #
20
+ # Example:
21
+ #
22
+ # SemanticLogger.add_appender(
23
+ # appender: :opensearch,
24
+ # url: 'http://localhost:9200'
25
+ # )
26
+ module SemanticLogger
27
+ module Appender
28
+ class Opensearch < ElasticsearchBase
29
+ private
30
+
31
+ def client_class
32
+ ::OpenSearch::Client
33
+ end
34
+ end
35
+ end
36
+ end
@@ -12,7 +12,7 @@ end
12
12
  # appender: :rabbitmq,
13
13
  #
14
14
  # # Name of the queue in RabbitMQ where to publish the logs. This queue will be bound to "amqp.direct" exchange.
15
- # queue: 'semantic_logger',
15
+ # queue_name: 'semantic_logger',
16
16
  #
17
17
  # # This host will be used for RabbitMQ connection.
18
18
  # # NOTE this is different than :host option which is used by the logger directly.
@@ -81,14 +81,15 @@ module SemanticLogger
81
81
  #
82
82
  # more parameters supported by Bunny: http://rubybunny.info/articles/connecting.html
83
83
  def initialize(queue_name: "semantic_logger", rabbitmq_host: nil,
84
- level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: true,
85
- **args, &block)
84
+ level: nil, formatter: nil, filter: nil, application: nil, environment: nil,
85
+ host: nil, metrics: true, **args, &)
86
86
  @queue_name = queue_name
87
87
  @rabbitmq_args = args.dup
88
88
  @rabbitmq_args[:host] = rabbitmq_host
89
89
  @rabbitmq_args[:logger] = logger
90
90
 
91
- super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &block)
91
+ super(level: level, formatter: formatter, filter: filter, application: application,
92
+ environment: environment, host: host, metrics: metrics, &)
92
93
  reopen
93
94
  end
94
95
 
@@ -39,6 +39,11 @@ module SemanticLogger
39
39
  # Name of this application to appear in log messages.
40
40
  # Default: SemanticLogger.application
41
41
  def initialize(level: :error, **args, &block)
42
+ Kernel.warn(
43
+ "SemanticLogger::Appender::Sentry (appender: :sentry) is deprecated and will be removed in v6. " \
44
+ "It depends on the end-of-life sentry-raven gem. Use appender: :sentry_ruby instead.",
45
+ category: :deprecated
46
+ )
42
47
  # Replace the Sentry Raven logger so that we can identify its log messages and not forward them to Sentry
43
48
  Raven.configure { |config| config.logger = SemanticLogger[Raven] }
44
49
  super
@@ -133,10 +133,10 @@ module SemanticLogger
133
133
  #
134
134
  def extract_tags!(context)
135
135
  named_tags = context.delete(:named_tags) || {}
136
- named_tags = named_tags.map { |k, v| [k.to_s, v.to_s] }.to_h
136
+ named_tags = named_tags.to_h { |k, v| [k.to_s, v.to_s] }
137
137
  tags = context.delete(:tags)
138
138
  named_tags.merge!("tag" => tags.join(", ")) { |_, v1, v2| "#{v1}, #{v2}" } if tags
139
- named_tags.map { |k, v| [k[0...32], v[0...256]] }.to_h
139
+ named_tags.to_h { |k, v| [k[0...32], v[0...256]] }
140
140
  end
141
141
  end
142
142
  end
@@ -89,11 +89,11 @@ module SemanticLogger
89
89
  # regular expression. All other messages will be ignored.
90
90
  # Proc: Only include log messages where the supplied Proc returns true
91
91
  # The Proc must return true or false.
92
- def initialize(index: "main", source_type: nil, **args, &block)
92
+ def initialize(index: "main", source_type: nil, **args, &)
93
93
  @index = index
94
94
  @source_type = source_type
95
95
 
96
- super(**args, &block)
96
+ super(**args, &)
97
97
  reopen
98
98
  end
99
99
 
@@ -73,11 +73,11 @@ module SemanticLogger
73
73
  index: nil,
74
74
  compress: true,
75
75
  **args,
76
- &block)
76
+ &)
77
77
  @source_type = source_type
78
78
  @index = index
79
79
 
80
- super(compress: compress, **args, &block)
80
+ super(compress: compress, **args, &)
81
81
 
82
82
  # Put splunk auth token in the header of every HTTP post.
83
83
  @header["Authorization"] = "Splunk #{token}"
@@ -98,7 +98,7 @@ module SemanticLogger
98
98
  }
99
99
  message[:sourcetype] = source_type if source_type
100
100
  message[:index] = index if index
101
- message.to_json
101
+ Utils.to_json(message)
102
102
  end
103
103
  end
104
104
  end
@@ -130,7 +130,7 @@ module SemanticLogger
130
130
  options: ::Syslog::LOG_PID | ::Syslog::LOG_CONS,
131
131
  tcp_client: {},
132
132
  **args,
133
- &block)
133
+ &)
134
134
  @options = options
135
135
  @facility = facility
136
136
  @max_size = max_size
@@ -151,7 +151,8 @@ module SemanticLogger
151
151
  require "syslog_protocol"
152
152
  rescue LoadError
153
153
  raise LoadError,
154
- "Missing gem: syslog_protocol. This gem is required when logging over TCP or UDP. To fix this error: gem install syslog_protocol"
154
+ "Missing gem: syslog_protocol. This gem is required when logging over TCP or UDP. " \
155
+ "To fix this error: gem install syslog_protocol"
155
156
  end
156
157
 
157
158
  # The net_tcp_client gem is required when logging over TCP.
@@ -160,12 +161,13 @@ module SemanticLogger
160
161
  require "net/tcp_client"
161
162
  rescue LoadError
162
163
  raise LoadError,
163
- "Missing gem: net_tcp_client. This gem is required when logging over TCP. To fix this error: gem install net_tcp_client"
164
+ "Missing gem: net_tcp_client. This gem is required when logging over TCP. " \
165
+ "To fix this error: gem install net_tcp_client"
164
166
  end
165
167
  end
166
168
  end
167
169
 
168
- super(**args, &block)
170
+ super(**args, &)
169
171
  reopen
170
172
  end
171
173
 
@@ -210,11 +212,13 @@ module SemanticLogger
210
212
  @remote_syslog.flush if @remote_syslog.respond_to?(:flush)
211
213
  end
212
214
 
213
- # Returns [SemanticLogger::Formatters::Base] default formatter for this Appender depending on the protocal selected
215
+ # Returns [SemanticLogger::Formatters::Base] default formatter for this Appender
216
+ # depending on the protocol selected
214
217
  def default_formatter
215
218
  if protocol == :syslog
216
- # Format is text output without the time
217
- SemanticLogger::Formatters::Default.new(time_format: :notime)
219
+ # Format is text output without the time.
220
+ # Escape control characters so untrusted log data cannot forge syslog entries.
221
+ SemanticLogger::Formatters::Default.new(time_format: :notime, escape_control_chars: true)
218
222
  else
219
223
  SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
220
224
  end
@@ -183,8 +183,8 @@ module SemanticLogger
183
183
  # connect_retry_count: 5
184
184
  # )
185
185
  def initialize(separator: "\n",
186
- level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: false,
187
- **tcp_client_args, &block)
186
+ level: nil, formatter: nil, filter: nil, application: nil, environment: nil,
187
+ host: nil, metrics: false, **tcp_client_args, &)
188
188
  @separator = separator
189
189
  @tcp_client_args = tcp_client_args
190
190
 
@@ -192,7 +192,8 @@ module SemanticLogger
192
192
  Net::TCPClient.logger = logger
193
193
  Net::TCPClient.logger.name = "Net::TCPClient"
194
194
 
195
- super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &block)
195
+ super(level: level, formatter: formatter, filter: filter, application: application,
196
+ environment: environment, host: host, metrics: metrics, &)
196
197
  reopen
197
198
  end
198
199
 
@@ -61,11 +61,11 @@ module SemanticLogger
61
61
  # appender: :udp,
62
62
  # server: 'server:3300'
63
63
  # )
64
- def initialize(server:, udp_flags: 0, metrics: true, **args, &block)
64
+ def initialize(server:, udp_flags: 0, metrics: true, **args, &)
65
65
  @server = server
66
66
  @udp_flags = udp_flags
67
67
 
68
- super(metrics: metrics, **args, &block)
68
+ super(metrics: metrics, **args, &)
69
69
  reopen
70
70
  end
71
71
 
@@ -38,17 +38,18 @@ module SemanticLogger
38
38
  # logger.info('Hello World', some: :payload)
39
39
  #
40
40
  # Install the `rails_semantic_logger` gem to replace the Rails logger with Semantic Logger.
41
- def initialize(logger:, **args, &block)
41
+ def initialize(logger:, **args, &)
42
42
  @logger = logger
43
43
 
44
44
  # Check if the custom appender responds to all the log levels. For example Ruby ::Logger
45
- does_not_implement = LEVELS[1..-1].find { |i| !@logger.respond_to?(i) }
45
+ does_not_implement = LEVELS[1..].find { |i| !@logger.respond_to?(i) }
46
46
  if does_not_implement
47
47
  raise(ArgumentError,
48
- "Supplied logger does not implement:#{does_not_implement}. It must implement all of #{LEVELS[1..-1].inspect}")
48
+ "Supplied logger does not implement:#{does_not_implement}. " \
49
+ "It must implement all of #{LEVELS[1..].inspect}")
49
50
  end
50
51
 
51
- super(**args, &block)
52
+ super(**args, &)
52
53
  end
53
54
 
54
55
  # Pass log calls to the underlying Rails, log4j or Ruby logger
@@ -2,10 +2,10 @@ module SemanticLogger
2
2
  module Appender
3
3
  # @formatter:off
4
4
  autoload :Async, "semantic_logger/appender/async"
5
- autoload :AsyncBatch, "semantic_logger/appender/async_batch"
6
5
  autoload :Bugsnag, "semantic_logger/appender/bugsnag"
7
6
  autoload :CloudwatchLogs, "semantic_logger/appender/cloudwatch_logs"
8
7
  autoload :Elasticsearch, "semantic_logger/appender/elasticsearch"
8
+ autoload :ElasticsearchBase, "semantic_logger/appender/elasticsearch_base"
9
9
  autoload :ElasticsearchHttp, "semantic_logger/appender/elasticsearch_http"
10
10
  autoload :File, "semantic_logger/appender/file"
11
11
  autoload :Graylog, "semantic_logger/appender/graylog"
@@ -18,6 +18,7 @@ module SemanticLogger
18
18
  autoload :MongoDB, "semantic_logger/appender/mongodb"
19
19
  autoload :NewRelic, "semantic_logger/appender/new_relic"
20
20
  autoload :NewRelicLogs, "semantic_logger/appender/new_relic_logs"
21
+ autoload :Opensearch, "semantic_logger/appender/opensearch"
21
22
  autoload :OpenTelemetry, "semantic_logger/appender/open_telemetry"
22
23
  autoload :Rabbitmq, "semantic_logger/appender/rabbitmq"
23
24
  autoload :Splunk, "semantic_logger/appender/splunk"
@@ -34,27 +35,37 @@ module SemanticLogger
34
35
  def self.factory(async: false, batch: nil,
35
36
  max_queue_size: 10_000, lag_check_interval: 1_000, lag_threshold_s: 30,
36
37
  batch_size: 300, batch_seconds: 5,
38
+ non_blocking: false, dropped_message_report_seconds: 30,
39
+ async_max_retries: 100,
37
40
  **args,
38
- &block)
39
- appender = build(**args, &block)
41
+ &)
42
+ appender = build(**args, &)
40
43
 
41
- # If appender implements #batch, then it should use the batch proxy by default.
42
- batch = true if batch.nil? && appender.respond_to?(:batch)
44
+ # If appender implements #batch, then it should use the batch proxy by default,
45
+ # unless the appender opts out of batching by default (e.g. the HTTP appender).
46
+ batch = true if batch.nil? && appender.respond_to?(:batch) && appender.batch_by_default?
43
47
 
44
48
  if batch == true
45
- Appender::AsyncBatch.new(
46
- appender: appender,
47
- max_queue_size: max_queue_size,
48
- lag_threshold_s: lag_threshold_s,
49
- batch_size: batch_size,
50
- batch_seconds: batch_seconds
49
+ Appender::Async.new(
50
+ appender: appender,
51
+ batch: true,
52
+ max_queue_size: max_queue_size,
53
+ lag_threshold_s: lag_threshold_s,
54
+ batch_size: batch_size,
55
+ batch_seconds: batch_seconds,
56
+ non_blocking: non_blocking,
57
+ dropped_message_report_seconds: dropped_message_report_seconds,
58
+ async_max_retries: async_max_retries
51
59
  )
52
60
  elsif async == true
53
61
  Appender::Async.new(
54
- appender: appender,
55
- max_queue_size: max_queue_size,
56
- lag_check_interval: lag_check_interval,
57
- lag_threshold_s: lag_threshold_s
62
+ appender: appender,
63
+ max_queue_size: max_queue_size,
64
+ lag_check_interval: lag_check_interval,
65
+ lag_threshold_s: lag_threshold_s,
66
+ non_blocking: non_blocking,
67
+ dropped_message_report_seconds: dropped_message_report_seconds,
68
+ async_max_retries: async_max_retries
58
69
  )
59
70
  else
60
71
  appender
@@ -62,13 +73,13 @@ module SemanticLogger
62
73
  end
63
74
 
64
75
  # Returns [Subscriber] instance from the supplied options.
65
- def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &block)
76
+ def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &)
66
77
  if file_name
67
- SemanticLogger::Appender::File.new(file_name, **args, &block)
78
+ SemanticLogger::Appender::File.new(file_name, **args, &)
68
79
  elsif io
69
- SemanticLogger::Appender::IO.new(io, **args, &block)
80
+ SemanticLogger::Appender::IO.new(io, **args, &)
70
81
  elsif logger
71
- SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &block)
82
+ SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &)
72
83
  elsif appender
73
84
  if appender.is_a?(Symbol)
74
85
  SemanticLogger::Utils.constantize_symbol(appender).new(**args)
@@ -76,7 +87,8 @@ module SemanticLogger
76
87
  appender
77
88
  else
78
89
  raise(ArgumentError,
79
- "Parameter :appender must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
90
+ "Parameter :appender must be either a Symbol or an object derived from " \
91
+ "SemanticLogger::Subscriber, not: #{appender.inspect}")
80
92
  end
81
93
  elsif metric
82
94
  if metric.is_a?(Symbol)
@@ -85,11 +97,13 @@ module SemanticLogger
85
97
  metric
86
98
  else
87
99
  raise(ArgumentError,
88
- "Parameter :metric must be either a Symbol or an object derived from SemanticLogger::Subscriber, not: #{appender.inspect}")
100
+ "Parameter :metric must be either a Symbol or an object derived from " \
101
+ "SemanticLogger::Subscriber, not: #{appender.inspect}")
89
102
  end
90
103
  else
91
104
  raise(ArgumentError,
92
- "To create an appender it must supply one of the following: :io, :file_name, :appender, :metric, or :logger")
105
+ "To create an appender it must supply one of the following: " \
106
+ ":io, :file_name, :appender, :metric, or :logger")
93
107
  end
94
108
  end
95
109
 
@@ -9,11 +9,13 @@ module SemanticLogger
9
9
  super()
10
10
  end
11
11
 
12
- def add(**args, &block)
13
- appender = SemanticLogger::Appender.factory(**args, &block)
12
+ def add(**args, &)
13
+ appender = SemanticLogger::Appender.factory(**args, &)
14
14
 
15
- if appender.respond_to?(:console_output?) && appender.console_output? && console_output?
16
- logger.warn "Ignoring attempt to add a second console appender: #{appender.class.name} since it would result in duplicate console output."
15
+ stream = appender.respond_to?(:console_stream) && appender.console_stream
16
+ if stream && console_streams.include?(stream)
17
+ logger.warn "Ignoring attempt to add a second #{stream} console appender since it " \
18
+ "would result in duplicate console output."
17
19
  return
18
20
  end
19
21
 
@@ -21,10 +23,30 @@ module SemanticLogger
21
23
  appender
22
24
  end
23
25
 
26
+ # The console streams (:stdout and/or :stderr) already being written to by the existing appenders.
27
+ def console_streams
28
+ filter_map { |appender| appender.console_stream if appender.respond_to?(:console_stream) }
29
+ end
30
+
24
31
  # Whether any of the existing appenders already output to the console?
25
32
  # I.e. Writes to stdout or stderr.
26
33
  def console_output?
27
- any? { |appender| appender.respond_to?(:console_output?) && appender.console_output? }
34
+ console_streams.any?
35
+ end
36
+
37
+ # Returns [Array<Hash>] operational statistics for each appender.
38
+ #
39
+ # Appenders that run asynchronously (see SemanticLogger::Appender::Async#stats) report
40
+ # their queue size and processed/dropped counts. Appenders that log inline on the
41
+ # processor thread report only their name with `async: false`.
42
+ def stats
43
+ map do |appender|
44
+ if appender.respond_to?(:stats)
45
+ appender.stats
46
+ else
47
+ {name: appender.name, async: false}
48
+ end
49
+ end
28
50
  end
29
51
 
30
52
  def log(log)