semantic_logger 5.0.0 → 5.2.0

data/lib/semantic_logger/appender/async.rb

@@ -21,13 +21,14 @@ module SemanticLogger
       # Methods forwarded to the queue processor that owns the thread and queue.
       # Tuning options (max_queue_size, batch_size, non_blocking, ...) are set once at
       # construction via SemanticLogger.add_appender, so only the values that callers actually
-      # read back are re-exposed here. lag_* backs the public SemanticLogger.lag_* API.
+      # read back are re-exposed here. lag_* and non_blocking= back the public
+      # SemanticLogger.lag_* and SemanticLogger.non_blocking= API on the main Processor.
       def_delegators :@processor,
                      :log, :flush, :close, :thread, :active?, :queue, :max_queue_size,
-                     :capped?, :non_blocking?, :batch?,
+                     :capped?, :non_blocking?, :non_blocking=, :batch?,
                      :lag_check_interval, :lag_check_interval=,
                      :lag_threshold_s, :lag_threshold_s=,
-                     :processed_count, :dropped_count
+                     :processed_count, :dropped_count, :failed_count
 
       # Appender proxy to allow an existing appender to run asynchronously in a separate thread.
       #
@@ -73,7 +74,8 @@ module SemanticLogger
       #
       #   dropped_message_report_seconds: [Integer]
       #     When non_blocking is enabled, log the count of dropped messages to the internal logger
-      #     at most once every this number of seconds.
+      #     at most once every this number of seconds. Also limits how often messages discarded
+      #     because their content could not be written are reported, after the first.
       #     Default: 30
       #
       #   async_max_retries: [Integer]
@@ -87,6 +89,14 @@ module SemanticLogger
         @processor = QueueProcessor.start(appender: appender, **args)
       end
 
+      # Returns [String] the class and object address of this proxy, whether it batches, and the
+      # wrapped appender's own #inspect, which shows only what that appender says is safe, but none of
+      # the instance variables of this proxy or its queue processor. See Subscriber#inspect.
+      def inspect
+        address = Kernel.instance_method(:to_s).bind_call(self).delete_suffix(">")
+        "#{address} batch=#{batch?} appender=#{appender.inspect}>"
+      end
+
       # Re-open appender after a fork.
       def reopen
         appender.reopen if appender.respond_to?(:reopen)
@@ -104,6 +114,8 @@ module SemanticLogger
       #   processed:      [Integer] Cumulative number of log messages written since startup.
       #   dropped:        [Integer] Cumulative number of log messages dropped because the queue
       #                             was full (only possible when non_blocking is enabled).
+      #   failed:         [Integer] Cumulative number of log messages discarded because their
+      #                             content could not be written (see Subscriber#message_error?).
       def stats
         {
           name:           name,
@@ -113,7 +125,8 @@ module SemanticLogger
           capped:         capped?,
           max_queue_size: capped? ? max_queue_size : nil,
           processed:      processed_count,
-          dropped:        dropped_count
+          dropped:        dropped_count,
+          failed:         failed_count
         }
       end
     end

data/lib/semantic_logger/appender/bugsnag.rb

@@ -56,7 +56,7 @@ module SemanticLogger
 
         # Send error messages as Runtime exceptions
         exception = extract_exception(log)
-        hash      = formatter.call(log, self)
+        hash      = format_log(log)
         bugsnag_notify(exception, hash, log_level(log.level))
         true
       end

data/lib/semantic_logger/appender/cloudwatch_logs.rb

@@ -2,7 +2,8 @@ begin
   require "aws-sdk-cloudwatchlogs"
 rescue LoadError
   raise LoadError,
-        'Gem aws-sdk-cloudwatchlogs is required for logging to CloudWatch Logs. Please add the gem "aws-sdk-cloudwatchlogs" to your Gemfile.'
+        "Gem aws-sdk-cloudwatchlogs is required for logging to CloudWatch Logs. " \
+        'Please add the gem "aws-sdk-cloudwatchlogs" to your Gemfile.'
 end
 
 require "concurrent"
@@ -20,7 +21,15 @@ require "concurrent"
 module SemanticLogger
   module Appender
     class CloudwatchLogs < SemanticLogger::Subscriber
-      attr_reader :client_kwargs, :group, :create_group, :create_stream, :force_flush_interval_seconds, :max_buffered_events,
+      # PutLogEvents limits, from the AWS documentation: a batch holds at most MAX_BATCH_EVENTS
+      # events, and at most MAX_BATCH_BYTES counting each event's message in UTF-8 plus
+      # EVENT_OVERHEAD_BYTES, in chronological order.
+      MAX_BATCH_BYTES      = 1_048_576
+      MAX_BATCH_EVENTS     = 10_000
+      EVENT_OVERHEAD_BYTES = 26
+
+      attr_reader :client_kwargs, :group, :create_group, :create_stream,
+                  :force_flush_interval_seconds, :max_buffered_events,
                   :task, :client, :buffered_logs
 
       # Create CloudWatch Logs Appender
@@ -78,14 +87,19 @@ module SemanticLogger
       end
 
       # Method called to log an event
+      #
+      # Formats the log message now, so that one that cannot be formatted, or that CloudWatch Logs
+      # would reject, raises here for that message alone, and the buffer only holds log events
+      # ready to send.
       def log(log)
-        buffered_logs << log
-
+        buffered_logs << log_event(log)
         put_log_events if buffered_logs.size >= max_buffered_events
+        true
       end
 
+      # Sends the buffered log messages on the calling thread, rather than waiting for the timer.
       def flush
-        task.execute while buffered_logs.size.positive?
+        put_log_events while buffered_logs.size.positive?
       end
 
       def close
@@ -108,40 +122,123 @@ module SemanticLogger
         SemanticLogger::Formatters::Json.new
       end
 
+      # Only the log group and stream: the client options can hold AWS credentials. See
+      # Subscriber#inspect.
+      def inspect_details
+        "group=#{group.inspect} stream=#{stream.inspect}"
+      end
+
       private
 
+      # Returns [Hash] the PutLogEvents log event for the log message.
+      def log_event(log)
+        {timestamp: (log.time.to_f * 1000).floor, message: event_message(format_log(log))}
+      end
+
+      # Returns [String] the formatted log message in UTF-8, cut short to the most that a batch can
+      # hold.
+      #
+      # Raises FormatError for a message that CloudWatch Logs would reject, together with every
+      # other message in the same request: one that is not a String, or is empty.
+      def event_message(message)
+        unless message.is_a?(String)
+          raise(FormatError, "Could not format the log message (the formatter returned a #{message.class})")
+        end
+
+        message = Utils.encode_utf8_string(message)
+        raise(FormatError, "Could not format the log message (it is empty)") if message.empty?
+
+        limit = MAX_BATCH_BYTES - EVENT_OVERHEAD_BYTES
+        message.bytesize <= limit ? message : message.byteslice(0, limit).scrub("")
+      end
+
+      # Sends the buffered log events, in as many requests as the PutLogEvents limits need.
+      #
+      # A request that fails is reported to the internal logger rather than raised: this usually
+      # runs on the timer thread, which discards errors, and the log events were already taken off
+      # the buffer, so AWS rejecting a batch, for example one over the size limit, lost every
+      # message in it without a trace.
       def put_log_events
-        logs = buffered_logs.shift(max_buffered_events)
+        events = buffered_logs.shift(max_buffered_events)
+        return if events.none?
 
-        return if logs.none?
+        batches(events).each { |batch| put_batch(batch) }
+      end
 
+      # Returns [Array<Array<Hash>>] the log events in chronological order, as PutLogEvents requires,
+      # split into batches that it accepts.
+      def batches(events)
+        batches = [[]]
+        bytes   = 0
+        # Keeps the order of log events with the same timestamp.
+        events.each_with_index.sort_by { |event, index| [event[:timestamp], index] }.each do |event, _index|
+          size = event[:message].bytesize + EVENT_OVERHEAD_BYTES
+          if batches.last.size >= MAX_BATCH_EVENTS || bytes + size > MAX_BATCH_BYTES
+            batches << []
+            bytes = 0
+          end
+          batches.last << event
+          bytes += size
+        end
+        batches.reject(&:empty?)
+      end
+
+      # Sends one batch, reporting any failure to the internal logger, so that the other batches are
+      # still sent.
+      def put_batch(events)
+        response = send_batch(events)
+        report_rejected_events(events, response.rejected_log_events_info)
+      rescue StandardError => e
+        logger.error("Could not send #{events.size} log message(s) to CloudWatch Logs", e)
+      end
+
+      # Returns the PutLogEvents response, after creating the log group or stream when it is missing
+      # and this appender may create it.
+      def send_batch(events)
+        created = []
         begin
-          client.put_log_events({
-                                  log_group_name:  group,
-                                  log_stream_name: stream,
-                                  log_events:      logs.map do |log|
-                                                     {
-                                                       timestamp: (log.time.to_f * 1000).floor,
-                                                       message:   formatter.call(log, self)
-                                                     }
-                                                   end
-                                })
+          client.put_log_events(log_group_name: group, log_stream_name: stream, log_events: events)
         rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
-          if e.message.include?("log group does not exist.") && create_group
-            client.create_log_group({
-                                      log_group_name: group
-                                    })
-            retry
-          elsif e.message.include?("log stream does not exist.") && create_stream
-            client.create_log_stream({
-                                       log_group_name:  group,
-                                       log_stream_name: stream
-                                     })
-            retry
-          end
+          resource = missing_resource(e)
+          raise if resource.nil? || created.include?(resource)
+
+          create(resource)
+          created << resource
+          retry
         end
       end
 
+      # Returns [:group | :stream | nil] the missing resource that this appender may create.
+      def missing_resource(exception)
+        if exception.message.include?("log group does not exist.") && create_group
+          :group
+        elsif exception.message.include?("log stream does not exist.") && create_stream
+          :stream
+        end
+      end
+
+      def create(resource)
+        if resource == :group
+          client.create_log_group(log_group_name: group)
+        else
+          client.create_log_stream(log_group_name: group, log_stream_name: stream)
+        end
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        # Created meanwhile, for example by another process logging to the same log group.
+        nil
+      end
+
+      # PutLogEvents accepts a request in which it rejects some of the log events: those more than 2
+      # hours in the future, older than 14 days, or older than the log group's retention period.
+      def report_rejected_events(events, rejected)
+        return unless rejected
+
+        logger.warn(
+          "CloudWatch Logs rejected some of #{events.size} log message(s) as too new, too old, or expired",
+          rejected.to_h
+        )
+      end
+
       def stream
         @stream || host
       end

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -2,7 +2,8 @@ begin
   require "elasticsearch"
 rescue LoadError
   raise LoadError,
-        'Gem elasticsearch is required for logging to Elasticsearch. Please add the gem "elasticsearch" to your Gemfile.'
+        "Gem elasticsearch is required for logging to Elasticsearch. " \
+        'Please add the gem "elasticsearch" to your Gemfile.'
 end
 
 require "semantic_logger/appender/elasticsearch_base"
@@ -26,10 +27,6 @@ module SemanticLogger
       def client_class
         ::Elasticsearch::Client
       end
-
-      def version_supports_type?
-        Gem::Version.new(::Elasticsearch::VERSION) < Gem::Version.new(7)
-      end
     end
   end
 end

data/lib/semantic_logger/appender/elasticsearch_base.rb

@@ -6,14 +6,13 @@ module SemanticLogger
     #
     # Implements the shared bulk-indexing pipeline used by both the
     # {Elasticsearch} and {OpenSearch} appenders. Subclasses only need to
-    # supply the backing client class (and, optionally, whether the server
-    # version still supports document `_type`).
+    # supply the backing client class.
     #
     # This class is internal: applications add an appender via
     # `SemanticLogger.add_appender(appender: :elasticsearch, ...)` or
     # `appender: :opensearch`, never by referencing this class directly.
     class ElasticsearchBase < SemanticLogger::Subscriber
-      attr_accessor :url, :index, :date_pattern, :type, :client, :flush_interval, :timeout_interval, :batch_size,
+      attr_accessor :url, :index, :date_pattern, :client, :flush_interval, :timeout_interval, :batch_size,
                     :client_args
 
       # Create an Elasticsearch-compatible appender over persistent HTTP(S).
@@ -31,9 +30,9 @@ module SemanticLogger
       #     Default: '%Y.%m.%d'
       #
       #   type: [String]
-      #     Document type to associate with logs when they are written.
-      #     Deprecated in Elasticsearch 7.0.0, unused by OpenSearch.
-      #     Default: 'log'
+      #     Deprecated and ignored. Document `_type` has been unused since
+      #     Elasticsearch 7 and is ignored by OpenSearch. Passing it logs a
+      #     deprecation warning; it will be removed in v6.
       #
       #   level: [:trace | :debug | :info | :warn | :error | :fatal]
       #     Override the log level for this appender.
@@ -124,7 +123,7 @@ module SemanticLogger
       def initialize(url: "http://localhost:9200",
                      index: "semantic_logger",
                      date_pattern: "%Y.%m.%d",
-                     type: "log",
+                     type: nil,
                      level: nil,
                      formatter: nil,
                      filter: nil,
@@ -134,40 +133,60 @@ module SemanticLogger
                      data_stream: false,
                      **client_args,
                      &)
+        if type
+          Kernel.warn(
+            "The Elasticsearch/OpenSearch appender `type:` parameter is deprecated and will be removed in v6. " \
+            "Document `_type` has been unused since Elasticsearch 7 and is ignored by OpenSearch.",
+            category: :deprecated
+          )
+        end
+
         @url                  = url
         @index                = index
         @date_pattern         = date_pattern
-        @type                 = type
         @client_args          = client_args.dup
         @client_args[:url]    = url if url && !client_args[:hosts]
         @client_args[:logger] = logger
         @data_stream          = data_stream
 
-        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: false, &)
+        super(level: level, formatter: formatter, filter: filter, application: application,
+              environment: environment, host: host, metrics: false, &)
         reopen
       end
 
+      # The url is left to the client gem to parse, since it also takes a comma separated list of
+      # urls, an Array of them, or hosts without a scheme.
       def reopen
-        @client = client_class.new(@client_args)
+        @client = Utils.without_url_in_errors(option: ":url or :hosts") { client_class.new(@client_args) }
       end
 
       # Log to the index for today
       def log(log)
-        bulk_payload = formatter.call(log, self)
-        bulk_write([bulk_index(log), bulk_payload])
+        bulk_write([bulk_index(log), bulk_line(log)])
         true
       end
 
       def batch(logs)
         messages = []
         logs.each do |log|
-          messages << bulk_index(log) << formatter.call(log, self)
+          messages << bulk_index(log) << bulk_line(log)
         end
 
         bulk_write(messages)
         true
       end
 
+      # The whole batch is formatted into a single bulk request before anything is sent.
+      def batch_all_or_nothing?
+        true
+      end
+
+      # The url without its credentials, and the index. See Subscriber#inspect.
+      def inspect_details
+        url_detail = @client_args.key?(:url) ? "url=#{Utils.redacted_url(url).inspect} " : ""
+        "#{url_detail}index=#{index.inspect}"
+      end
+
       private
 
       # The backing client class, e.g. ::Elasticsearch::Client or ::OpenSearch::Client.
@@ -175,12 +194,25 @@ module SemanticLogger
         raise NotImplementedError, "#{self.class} must implement #client_class"
       end
 
+      # Returns [String] the log message formatted as a bulk line, serialized by Utils.to_json
+      # and handed to the client as a String. The client gem's serializer writes Infinity and NaN
+      # as bare words, which is not JSON, and does not repair invalid UTF-8, a structure nested too
+      # deeply, or a Struct (written through its #inspect), so such an entry was dropped or its
+      # objects' instance variables were written. Serializing is part of formatting, so an error
+      # doing so discards the message rather than stalling the appender (see #format_log). A
+      # formatter that returns a String is taken to have serialized the line itself.
+      def bulk_line(log)
+        format_log(log) { |document| document.is_a?(String) ? document : Utils.to_json(document) }
+      end
+
+      # Serializes the action lines; the documents already are (see #bulk_line).
       def bulk_write(messages)
+        body        = messages.map { |message| message.is_a?(String) ? message : message.to_json }
         bulk_result =
           if @data_stream
-            @client.bulk(index: index, body: messages)
+            @client.bulk(index: index, body: body)
           else
-            @client.bulk(body: messages)
+            @client.bulk(body: body)
           end
 
         return unless bulk_result["errors"]
@@ -193,20 +225,13 @@ module SemanticLogger
         expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
         return {"create" => {}} if @data_stream
 
-        bulk_index = {"index" => {"_index" => expanded_index_name}}
-        bulk_index["index"].merge!({"_type" => type}) if version_supports_type?
-        bulk_index
+        {"index" => {"_index" => expanded_index_name}}
       end
 
       def default_formatter
         time_key = @data_stream ? "@timestamp" : :timestamp
         SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
       end
-
-      # Modern Elasticsearch (>= 7) and OpenSearch no longer support document `_type`.
-      def version_supports_type?
-        false
-      end
     end
   end
 end

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -13,7 +13,7 @@ require "date"
 module SemanticLogger
   module Appender
     class ElasticsearchHttp < SemanticLogger::Appender::Http
-      attr_accessor :index, :type
+      attr_accessor :index
 
       # Create Elasticsearch appender over persistent HTTP(S)
       #
@@ -25,9 +25,9 @@ module SemanticLogger
       #     Default: 'semantic_logger'
       #
       #   type: [String]
-      #     Document type to associate with logs when they are written.
-      #     Deprecated in Elasticsearch 7.0.0
-      #     Default: 'log'
+      #     Deprecated and ignored. Document `_type` has been unused since
+      #     Elasticsearch 7. Passing it logs a deprecation warning; it will be
+      #     removed in v6.
       #
       #   level: [:trace | :debug | :info | :warn | :error | :fatal]
       #     Override the log level for this appender.
@@ -52,21 +52,28 @@ module SemanticLogger
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
       def initialize(index: "semantic_logger",
-                     type: "log",
+                     type: nil,
                      url: "http://localhost:9200",
                      **http_args,
                      &)
+        if type
+          Kernel.warn(
+            "The Elasticsearch/OpenSearch appender `type:` parameter is deprecated and will be removed in v6. " \
+            "Document `_type` has been unused since Elasticsearch 7 and is ignored by OpenSearch.",
+            category: :deprecated
+          )
+        end
+
         @index = index
-        @type  = type
         super(url: url, **http_args, &)
 
         @request_path = "#{@path.end_with?('/') ? @path : "#{@path}/"}#{@index}-%Y.%m.%d"
-        @logging_path = "#{@request_path}/#{type}"
+        @logging_path = "#{@request_path}/_doc"
       end
 
       # Log to the index for today.
       def log(log)
-        post(formatter.call(log, self), log.time.strftime(@logging_path))
+        post(format_log(log), log.time.strftime(@logging_path))
       end
 
       # Deletes all log data captured for a day.