semantic_logger 4.17.0 → 5.0.0

data/lib/semantic_logger/appender/new_relic_logs.rb

@@ -23,6 +23,11 @@ require "semantic_logger/formatters/new_relic_logs"
 module SemanticLogger
   module Appender
     class NewRelicLogs < SemanticLogger::Subscriber
+      CAPTURE_CONTEXT = lambda do |log|
+        meta = ::NewRelic::Agent.linking_metadata
+        log.set_context(:new_relic_metadata, meta) unless meta.empty?
+      end
+
       # Create Appender
       #
       # Parameters
@@ -42,14 +47,19 @@ module SemanticLogger
       #           The Proc must return true or false.
       def initialize(formatter: SemanticLogger::Formatters::NewRelicLogs.new, **args, &block)
         super
+
+        # Record NewRelic's "trace.id"/"entity.name"/"hostname"/etc, so we can include them later in the formatted output.
+        # These are thread-local, so need to be captured as soon as the log-message is created.
+        # https://rubydoc.info/gems/newrelic_rpm/NewRelic/Agent#linking_metadata-instance_method
+        SemanticLogger.on_log(CAPTURE_CONTEXT)
       end
 
       # Send an error notification to New Relic
       def log(log)
         begin
           message = formatter.call(log, self) # Generate the structured log
-          json_message = message.to_json      # Convert the log to JSON
-          level = log.level.to_s.upcase       # Determine the log level
+          json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
+          level = log.level.to_s.upcase # Determine the log level
           self.class.log_newrelic(json_message, level)
         rescue JSON::GeneratorError => e
           warn("Failed to serialize log message to JSON: #{e.message}")

data/lib/semantic_logger/appender/open_telemetry.rb

@@ -12,7 +12,7 @@ end
 module SemanticLogger
   module Appender
     class OpenTelemetry < SemanticLogger::Subscriber
-      attr_reader :name, :version, :logger
+      attr_reader :name, :version, :provider, :logger
 
       CAPTURE_CONTEXT = ->(log) { log.set_context(:open_telemetry, ::OpenTelemetry::Context.current) }
 
@@ -29,16 +29,19 @@ module SemanticLogger
                      formatter: SemanticLogger::Formatters::OpenTelemetry.new,
                      metrics: true,
                      **args,
-                     &block)
-        @name    = name
-        @version = version
-        @logger  = ::OpenTelemetry.logger_provider.logger(name: @name, version: @version)
+                     &)
+        # Base#initialize (via Subscriber) overwrites @name with the class name,
+        # so call super first and then assign our own attributes.
+        super(formatter: formatter, metrics: metrics, **args, &)
+
+        @name      = name
+        @version   = version
+        @provider  = ::OpenTelemetry.logger_provider
+        @logger    = @provider.logger(name: @name, version: @version)
 
         # Capture the current Open Telemetry context when a log entry is captured.
         # Prevents duplicate subscribers as long as it is from a constant.
         SemanticLogger.on_log(CAPTURE_CONTEXT)
-
-        super(formatter: formatter, metrics: metrics, **args, &block)
       end
 
       def log(log)
@@ -56,19 +59,30 @@ module SemanticLogger
           timestamp:       time,
           body:            body.transform_keys!(&:to_s),
           attributes:      payload,
-          context:         log.context[:open_telemetry] || ::OpenTelemetry::Context.current
+          context:         (log.context && log.context[:open_telemetry]) || ::OpenTelemetry::Context.current
         )
         true
       end
 
       # Flush all pending logs.
       def flush
-        @logger.logger_provider.force_flush
+        return unless @provider
+
+        @provider.force_flush if @provider.respond_to?(:force_flush)
+      rescue StandardError => e
+        # Swallow to avoid noisy shutdown exceptions.
+        SemanticLogger::Processor.logger.warn("Flush failed: #{e.class}: #{e.message}")
       end
 
-      # Flush pending logs and close the appender
+      # Close the appender and release resources.
       def close
-        @logger.logger_provider.shutdown
+        return unless @provider
+
+        @provider.shutdown if @provider.respond_to?(:shutdown)
+      rescue StandardError => e
+        SemanticLogger::Processor.logger.warn("Shutdown failed: #{e.class}: #{e.message}")
+      ensure
+        @provider = nil
       end
 
       # For logging metrics only log events.

data/lib/semantic_logger/appender/opensearch.rb

@@ -0,0 +1,35 @@
+begin
+  require "opensearch-ruby"
+rescue LoadError
+  raise LoadError,
+        'Gem opensearch-ruby is required for logging to OpenSearch. Please add the gem "opensearch-ruby" to your Gemfile.'
+end
+
+require "semantic_logger/appender/elasticsearch_base"
+
+# Forward all log messages to OpenSearch (e.g. AWS OpenSearch).
+#
+# OpenSearch is a fork of Elasticsearch 7.10. The client API and bulk
+# protocol are identical, so this appender reuses the same pipeline as the
+# Elasticsearch appender; only the backing client gem differs. Use this
+# appender (with the `opensearch-ruby` gem) instead of `:elasticsearch`
+# when talking to an OpenSearch server, since modern `elasticsearch` gems
+# reject non-Elasticsearch servers via a product check.
+#
+# Example:
+#
+#   SemanticLogger.add_appender(
+#     appender: :opensearch,
+#     url:      'http://localhost:9200'
+#   )
+module SemanticLogger
+  module Appender
+    class Opensearch < ElasticsearchBase
+      private
+
+      def client_class
+        ::OpenSearch::Client
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/rabbitmq.rb

@@ -12,7 +12,7 @@ end
 #     appender: :rabbitmq,
 #
 #     # Name of the queue in RabbitMQ where to publish the logs. This queue will be bound to "amqp.direct" exchange.
-#     queue: 'semantic_logger',
+#     queue_name: 'semantic_logger',
 #
 #     # This host will be used for RabbitMQ connection.
 #     # NOTE this is different than :host option which is used by the logger directly.
@@ -82,13 +82,13 @@ module SemanticLogger
       #   more parameters supported by Bunny: http://rubybunny.info/articles/connecting.html
       def initialize(queue_name: "semantic_logger", rabbitmq_host: nil,
                      level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: true,
-                     **args, &block)
+                     **args, &)
         @queue_name             = queue_name
         @rabbitmq_args          = args.dup
         @rabbitmq_args[:host]   = rabbitmq_host
         @rabbitmq_args[:logger] = logger
 
-        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &block)
+        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &)
         reopen
       end
 

data/lib/semantic_logger/appender/sentry_ruby.rb

@@ -40,7 +40,12 @@ module SemanticLogger
       def initialize(level: :error, **args, &block)
         # Replace the Sentry Ruby logger so that we can identify its log
         # messages and not forward them to Sentry
-        ::Sentry.init { |config| config.logger = SemanticLogger[::Sentry] } unless ::Sentry.initialized?
+        unless ::Sentry.initialized?
+          ::Sentry.init do |config|
+            logger = SemanticLogger[::Sentry]
+            config.respond_to?(:sdk_logger=) ? config.sdk_logger = logger : config.logger = logger
+          end
+        end
         super
       end
 
@@ -128,10 +133,10 @@ module SemanticLogger
       #
       def extract_tags!(context)
         named_tags = context.delete(:named_tags) || {}
-        named_tags = named_tags.map { |k, v| [k.to_s, v.to_s] }.to_h
+        named_tags = named_tags.to_h { |k, v| [k.to_s, v.to_s] }
         tags = context.delete(:tags)
         named_tags.merge!("tag" => tags.join(", ")) { |_, v1, v2| "#{v1}, #{v2}" } if tags
-        named_tags.map { |k, v| [k[0...32], v[0...256]] }.to_h
+        named_tags.to_h { |k, v| [k[0...32], v[0...256]] }
       end
     end
   end

data/lib/semantic_logger/appender/splunk.rb

@@ -89,11 +89,11 @@ module SemanticLogger
       #     regular expression. All other messages will be ignored.
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
-      def initialize(index: "main", source_type: nil, **args, &block)
+      def initialize(index: "main", source_type: nil, **args, &)
         @index       = index
         @source_type = source_type
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 

data/lib/semantic_logger/appender/splunk_http.rb

@@ -73,11 +73,11 @@ module SemanticLogger
                      index: nil,
                      compress: true,
                      **args,
-                     &block)
+                     &)
         @source_type = source_type
         @index       = index
 
-        super(compress: compress, **args, &block)
+        super(compress: compress, **args, &)
 
         # Put splunk auth token in the header of every HTTP post.
         @header["Authorization"] = "Splunk #{token}"
@@ -98,7 +98,7 @@ module SemanticLogger
         }
         message[:sourcetype]  = source_type if source_type
         message[:index]       = index if index
-        message.to_json
+        Utils.to_json(message)
       end
     end
   end

data/lib/semantic_logger/appender/syslog.rb

@@ -130,7 +130,7 @@ module SemanticLogger
                      options: ::Syslog::LOG_PID | ::Syslog::LOG_CONS,
                      tcp_client: {},
                      **args,
-                     &block)
+                     &)
         @options            = options
         @facility           = facility
         @max_size           = max_size
@@ -165,7 +165,7 @@ module SemanticLogger
           end
         end
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 
@@ -213,8 +213,9 @@ module SemanticLogger
       # Returns [SemanticLogger::Formatters::Base] default formatter for this Appender depending on the protocal selected
       def default_formatter
         if protocol == :syslog
-          # Format is text output without the time
-          SemanticLogger::Formatters::Default.new(time_format: :notime)
+          # Format is text output without the time.
+          # Escape control characters so untrusted log data cannot forge syslog entries.
+          SemanticLogger::Formatters::Default.new(time_format: :notime, escape_control_chars: true)
         else
           SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
         end

data/lib/semantic_logger/appender/tcp.rb

@@ -184,7 +184,7 @@ module SemanticLogger
       #   )
       def initialize(separator: "\n",
                      level: nil, formatter: nil, filter: nil, application: nil, environment: nil, host: nil, metrics: false,
-                     **tcp_client_args, &block)
+                     **tcp_client_args, &)
         @separator       = separator
         @tcp_client_args = tcp_client_args
 
@@ -192,7 +192,7 @@ module SemanticLogger
         Net::TCPClient.logger      = logger
         Net::TCPClient.logger.name = "Net::TCPClient"
 
-        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &block)
+        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: metrics, &)
         reopen
       end
 

data/lib/semantic_logger/appender/udp.rb

@@ -61,11 +61,11 @@ module SemanticLogger
       #     appender: :udp,
       #     server:   'server:3300'
       #   )
-      def initialize(server:, udp_flags: 0, metrics: true, **args, &block)
+      def initialize(server:, udp_flags: 0, metrics: true, **args, &)
         @server    = server
         @udp_flags = udp_flags
 
-        super(metrics: metrics, **args, &block)
+        super(metrics: metrics, **args, &)
         reopen
       end
 

data/lib/semantic_logger/appender/wrapper.rb

@@ -38,17 +38,17 @@ module SemanticLogger
       #    logger.info('Hello World', some: :payload)
       #
       # Install the `rails_semantic_logger` gem to replace the Rails logger with Semantic Logger.
-      def initialize(logger:, **args, &block)
+      def initialize(logger:, **args, &)
         @logger = logger
 
         # Check if the custom appender responds to all the log levels. For example Ruby ::Logger
-        does_not_implement = LEVELS[1..-1].find { |i| !@logger.respond_to?(i) }
+        does_not_implement = LEVELS[1..].find { |i| !@logger.respond_to?(i) }
         if does_not_implement
           raise(ArgumentError,
-                "Supplied logger does not implement:#{does_not_implement}. It must implement all of #{LEVELS[1..-1].inspect}")
+                "Supplied logger does not implement:#{does_not_implement}. It must implement all of #{LEVELS[1..].inspect}")
         end
 
-        super(**args, &block)
+        super(**args, &)
       end
 
       # Pass log calls to the underlying Rails, log4j or Ruby logger

data/lib/semantic_logger/appender.rb

@@ -2,10 +2,10 @@ module SemanticLogger
   module Appender
     # @formatter:off
     autoload :Async,               "semantic_logger/appender/async"
-    autoload :AsyncBatch,          "semantic_logger/appender/async_batch"
     autoload :Bugsnag,             "semantic_logger/appender/bugsnag"
     autoload :CloudwatchLogs,      "semantic_logger/appender/cloudwatch_logs"
     autoload :Elasticsearch,       "semantic_logger/appender/elasticsearch"
+    autoload :ElasticsearchBase,   "semantic_logger/appender/elasticsearch_base"
     autoload :ElasticsearchHttp,   "semantic_logger/appender/elasticsearch_http"
     autoload :File,                "semantic_logger/appender/file"
     autoload :Graylog,             "semantic_logger/appender/graylog"
@@ -18,6 +18,7 @@ module SemanticLogger
     autoload :MongoDB,             "semantic_logger/appender/mongodb"
     autoload :NewRelic,            "semantic_logger/appender/new_relic"
     autoload :NewRelicLogs,        "semantic_logger/appender/new_relic_logs"
+    autoload :Opensearch,          "semantic_logger/appender/opensearch"
     autoload :OpenTelemetry,       "semantic_logger/appender/open_telemetry"
     autoload :Rabbitmq,            "semantic_logger/appender/rabbitmq"
     autoload :Splunk,              "semantic_logger/appender/splunk"
@@ -34,27 +35,37 @@ module SemanticLogger
     def self.factory(async: false, batch: nil,
                      max_queue_size: 10_000, lag_check_interval: 1_000, lag_threshold_s: 30,
                      batch_size: 300, batch_seconds: 5,
+                     non_blocking: false, dropped_message_report_seconds: 30,
+                     async_max_retries: 100,
                      **args,
-                     &block)
-      appender = build(**args, &block)
+                     &)
+      appender = build(**args, &)
 
-      # If appender implements #batch, then it should use the batch proxy by default.
-      batch    = true if batch.nil? && appender.respond_to?(:batch)
+      # If appender implements #batch, then it should use the batch proxy by default,
+      # unless the appender opts out of batching by default (e.g. the HTTP appender).
+      batch    = true if batch.nil? && appender.respond_to?(:batch) && appender.batch_by_default?
 
       if batch == true
-        Appender::AsyncBatch.new(
-          appender:        appender,
-          max_queue_size:  max_queue_size,
-          lag_threshold_s: lag_threshold_s,
-          batch_size:      batch_size,
-          batch_seconds:   batch_seconds
+        Appender::Async.new(
+          appender:                       appender,
+          batch:                          true,
+          max_queue_size:                 max_queue_size,
+          lag_threshold_s:                lag_threshold_s,
+          batch_size:                     batch_size,
+          batch_seconds:                  batch_seconds,
+          non_blocking:                   non_blocking,
+          dropped_message_report_seconds: dropped_message_report_seconds,
+          async_max_retries:              async_max_retries
         )
       elsif async == true
         Appender::Async.new(
-          appender:           appender,
-          max_queue_size:     max_queue_size,
-          lag_check_interval: lag_check_interval,
-          lag_threshold_s:    lag_threshold_s
+          appender:                       appender,
+          max_queue_size:                 max_queue_size,
+          lag_check_interval:             lag_check_interval,
+          lag_threshold_s:                lag_threshold_s,
+          non_blocking:                   non_blocking,
+          dropped_message_report_seconds: dropped_message_report_seconds,
+          async_max_retries:              async_max_retries
         )
       else
         appender
@@ -62,13 +73,13 @@ module SemanticLogger
     end
 
     # Returns [Subscriber] instance from the supplied options.
-    def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &block)
+    def self.build(io: nil, file_name: nil, appender: nil, metric: nil, logger: nil, **args, &)
       if file_name
-        SemanticLogger::Appender::File.new(file_name, **args, &block)
+        SemanticLogger::Appender::File.new(file_name, **args, &)
       elsif io
-        SemanticLogger::Appender::IO.new(io, **args, &block)
+        SemanticLogger::Appender::IO.new(io, **args, &)
       elsif logger
-        SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &block)
+        SemanticLogger::Appender::Wrapper.new(logger: logger, **args, &)
       elsif appender
         if appender.is_a?(Symbol)
           SemanticLogger::Utils.constantize_symbol(appender).new(**args)

data/lib/semantic_logger/appenders.rb

@@ -9,11 +9,12 @@ module SemanticLogger
       super()
     end
 
-    def add(**args, &block)
-      appender = SemanticLogger::Appender.factory(**args, &block)
+    def add(**args, &)
+      appender = SemanticLogger::Appender.factory(**args, &)
 
-      if appender.respond_to?(:console_output?) && appender.console_output? && console_output?
-        logger.warn "Ignoring attempt to add a second console appender: #{appender.class.name} since it would result in duplicate console output."
+      stream = appender.respond_to?(:console_stream) && appender.console_stream
+      if stream && console_streams.include?(stream)
+        logger.warn "Ignoring attempt to add a second #{stream} console appender since it would result in duplicate console output."
         return
       end
 
@@ -21,10 +22,30 @@ module SemanticLogger
       appender
     end
 
+    # The console streams (:stdout and/or :stderr) already being written to by the existing appenders.
+    def console_streams
+      filter_map { |appender| appender.console_stream if appender.respond_to?(:console_stream) }
+    end
+
     # Whether any of the existing appenders already output to the console?
     # I.e. Writes to stdout or stderr.
     def console_output?
-      any? { |appender| appender.respond_to?(:console_output?) && appender.console_output? }
+      console_streams.any?
+    end
+
+    # Returns [Array<Hash>] operational statistics for each appender.
+    #
+    # Appenders that run asynchronously (see SemanticLogger::Appender::Async#stats) report
+    # their queue size and processed/dropped counts. Appenders that log inline on the
+    # processor thread report only their name with `async: false`.
+    def stats
+      map do |appender|
+        if appender.respond_to?(:stats)
+          appender.stats
+        else
+          {name: appender.name, async: false}
+        end
+      end
     end
 
     def log(log)