semantic_logger 5.1.0 → 5.2.0

data/lib/semantic_logger/appender/bugsnag.rb

@@ -56,7 +56,7 @@ module SemanticLogger
 
         # Send error messages as Runtime exceptions
         exception = extract_exception(log)
-        hash      = formatter.call(log, self)
+        hash      = format_log(log)
         bugsnag_notify(exception, hash, log_level(log.level))
         true
       end

data/lib/semantic_logger/appender/cloudwatch_logs.rb

@@ -21,6 +21,13 @@ require "concurrent"
 module SemanticLogger
   module Appender
     class CloudwatchLogs < SemanticLogger::Subscriber
+      # PutLogEvents limits, from the AWS documentation: a batch holds at most MAX_BATCH_EVENTS
+      # events, and at most MAX_BATCH_BYTES counting each event's message in UTF-8 plus
+      # EVENT_OVERHEAD_BYTES, in chronological order.
+      MAX_BATCH_BYTES      = 1_048_576
+      MAX_BATCH_EVENTS     = 10_000
+      EVENT_OVERHEAD_BYTES = 26
+
       attr_reader :client_kwargs, :group, :create_group, :create_stream,
                   :force_flush_interval_seconds, :max_buffered_events,
                   :task, :client, :buffered_logs
@@ -80,14 +87,19 @@ module SemanticLogger
       end
 
       # Method called to log an event
+      #
+      # Formats the log message now, so that one that cannot be formatted, or that CloudWatch Logs
+      # would reject, raises here for that message alone, and the buffer only holds log events
+      # ready to send.
       def log(log)
-        buffered_logs << log
-
+        buffered_logs << log_event(log)
         put_log_events if buffered_logs.size >= max_buffered_events
+        true
       end
 
+      # Sends the buffered log messages on the calling thread, rather than waiting for the timer.
       def flush
-        task.execute while buffered_logs.size.positive?
+        put_log_events while buffered_logs.size.positive?
       end
 
       def close
@@ -110,38 +122,121 @@ module SemanticLogger
         SemanticLogger::Formatters::Json.new
       end
 
+      # Only the log group and stream: the client options can hold AWS credentials. See
+      # Subscriber#inspect.
+      def inspect_details
+        "group=#{group.inspect} stream=#{stream.inspect}"
+      end
+
       private
 
+      # Returns [Hash] the PutLogEvents log event for the log message.
+      def log_event(log)
+        {timestamp: (log.time.to_f * 1000).floor, message: event_message(format_log(log))}
+      end
+
+      # Returns [String] the formatted log message in UTF-8, cut short to the most that a batch can
+      # hold.
+      #
+      # Raises FormatError for a message that CloudWatch Logs would reject, together with every
+      # other message in the same request: one that is not a String, or is empty.
+      def event_message(message)
+        unless message.is_a?(String)
+          raise(FormatError, "Could not format the log message (the formatter returned a #{message.class})")
+        end
+
+        message = Utils.encode_utf8_string(message)
+        raise(FormatError, "Could not format the log message (it is empty)") if message.empty?
+
+        limit = MAX_BATCH_BYTES - EVENT_OVERHEAD_BYTES
+        message.bytesize <= limit ? message : message.byteslice(0, limit).scrub("")
+      end
+
+      # Sends the buffered log events, in as many requests as the PutLogEvents limits need.
+      #
+      # A request that fails is reported to the internal logger rather than raised: this usually
+      # runs on the timer thread, which discards errors, and the log events were already taken off
+      # the buffer, so AWS rejecting a batch, for example one over the size limit, lost every
+      # message in it without a trace.
       def put_log_events
-        logs = buffered_logs.shift(max_buffered_events)
+        events = buffered_logs.shift(max_buffered_events)
+        return if events.none?
+
+        batches(events).each { |batch| put_batch(batch) }
+      end
 
-        return if logs.none?
+      # Returns [Array<Array<Hash>>] the log events in chronological order, as PutLogEvents requires,
+      # split into batches that it accepts.
+      def batches(events)
+        batches = [[]]
+        bytes   = 0
+        # Keeps the order of log events with the same timestamp.
+        events.each_with_index.sort_by { |event, index| [event[:timestamp], index] }.each do |event, _index|
+          size = event[:message].bytesize + EVENT_OVERHEAD_BYTES
+          if batches.last.size >= MAX_BATCH_EVENTS || bytes + size > MAX_BATCH_BYTES
+            batches << []
+            bytes = 0
+          end
+          batches.last << event
+          bytes += size
+        end
+        batches.reject(&:empty?)
+      end
+
+      # Sends one batch, reporting any failure to the internal logger, so that the other batches are
+      # still sent.
+      def put_batch(events)
+        response = send_batch(events)
+        report_rejected_events(events, response.rejected_log_events_info)
+      rescue StandardError => e
+        logger.error("Could not send #{events.size} log message(s) to CloudWatch Logs", e)
+      end
 
+      # Returns the PutLogEvents response, after creating the log group or stream when it is missing
+      # and this appender may create it.
+      def send_batch(events)
+        created = []
         begin
-          client.put_log_events({
-                                  log_group_name:  group,
-                                  log_stream_name: stream,
-                                  log_events:      logs.map do |log|
-                                                     {
-                                                       timestamp: (log.time.to_f * 1000).floor,
-                                                       message:   formatter.call(log, self)
-                                                     }
-                                                   end
-                                })
+          client.put_log_events(log_group_name: group, log_stream_name: stream, log_events: events)
         rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
-          if e.message.include?("log group does not exist.") && create_group
-            client.create_log_group({
-                                      log_group_name: group
-                                    })
-            retry
-          elsif e.message.include?("log stream does not exist.") && create_stream
-            client.create_log_stream({
-                                       log_group_name:  group,
-                                       log_stream_name: stream
-                                     })
-            retry
-          end
+          resource = missing_resource(e)
+          raise if resource.nil? || created.include?(resource)
+
+          create(resource)
+          created << resource
+          retry
+        end
+      end
+
+      # Returns [:group | :stream | nil] the missing resource that this appender may create.
+      def missing_resource(exception)
+        if exception.message.include?("log group does not exist.") && create_group
+          :group
+        elsif exception.message.include?("log stream does not exist.") && create_stream
+          :stream
+        end
+      end
+
+      def create(resource)
+        if resource == :group
+          client.create_log_group(log_group_name: group)
+        else
+          client.create_log_stream(log_group_name: group, log_stream_name: stream)
         end
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        # Created meanwhile, for example by another process logging to the same log group.
+        nil
+      end
+
+      # PutLogEvents accepts a request in which it rejects some of the log events: those more than 2
+      # hours in the future, older than 14 days, or older than the log group's retention period.
+      def report_rejected_events(events, rejected)
+        return unless rejected
+
+        logger.warn(
+          "CloudWatch Logs rejected some of #{events.size} log message(s) as too new, too old, or expired",
+          rejected.to_h
+        )
       end
 
       def stream

data/lib/semantic_logger/appender/elasticsearch_base.rb

@@ -154,27 +154,39 @@ module SemanticLogger
         reopen
       end
 
+      # The url is left to the client gem to parse, since it also takes a comma separated list of
+      # urls, an Array of them, or hosts without a scheme.
       def reopen
-        @client = client_class.new(@client_args)
+        @client = Utils.without_url_in_errors(option: ":url or :hosts") { client_class.new(@client_args) }
       end
 
       # Log to the index for today
       def log(log)
-        bulk_payload = formatter.call(log, self)
-        bulk_write([bulk_index(log), bulk_payload])
+        bulk_write([bulk_index(log), bulk_line(log)])
         true
       end
 
       def batch(logs)
         messages = []
         logs.each do |log|
-          messages << bulk_index(log) << formatter.call(log, self)
+          messages << bulk_index(log) << bulk_line(log)
         end
 
         bulk_write(messages)
         true
       end
 
+      # The whole batch is formatted into a single bulk request before anything is sent.
+      def batch_all_or_nothing?
+        true
+      end
+
+      # The url without its credentials, and the index. See Subscriber#inspect.
+      def inspect_details
+        url_detail = @client_args.key?(:url) ? "url=#{Utils.redacted_url(url).inspect} " : ""
+        "#{url_detail}index=#{index.inspect}"
+      end
+
       private
 
       # The backing client class, e.g. ::Elasticsearch::Client or ::OpenSearch::Client.
@@ -182,12 +194,25 @@ module SemanticLogger
         raise NotImplementedError, "#{self.class} must implement #client_class"
       end
 
+      # Returns [String] the log message formatted as a bulk line, serialized by Utils.to_json
+      # and handed to the client as a String. The client gem's serializer writes Infinity and NaN
+      # as bare words, which is not JSON, and does not repair invalid UTF-8, a structure nested too
+      # deeply, or a Struct (written through its #inspect), so such an entry was dropped or its
+      # objects' instance variables were written. Serializing is part of formatting, so an error
+      # doing so discards the message rather than stalling the appender (see #format_log). A
+      # formatter that returns a String is taken to have serialized the line itself.
+      def bulk_line(log)
+        format_log(log) { |document| document.is_a?(String) ? document : Utils.to_json(document) }
+      end
+
+      # Serializes the action lines; the documents already are (see #bulk_line).
       def bulk_write(messages)
+        body        = messages.map { |message| message.is_a?(String) ? message : message.to_json }
         bulk_result =
           if @data_stream
-            @client.bulk(index: index, body: messages)
+            @client.bulk(index: index, body: body)
           else
-            @client.bulk(body: messages)
+            @client.bulk(body: body)
           end
 
         return unless bulk_result["errors"]

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -73,7 +73,7 @@ module SemanticLogger
 
       # Log to the index for today.
       def log(log)
-        post(formatter.call(log, self), log.time.strftime(@logging_path))
+        post(format_log(log), log.time.strftime(@logging_path))
       end
 
       # Deletes all log data captured for a day.

data/lib/semantic_logger/appender/file.rb

@@ -10,6 +10,15 @@ module SemanticLogger
                     :reopen_period, :reopen_count, :reopen_size, :permissions
       attr_reader :log_count, :log_size, :current_file_name, :reopen_at
 
+      # With permissions, the log file is opened without following a symbolic link, so that a link
+      # planted in the log directory can neither create nor open the file it points at, without
+      # blocking, so that a planted FIFO fails to open instead of waiting for a reader, and without
+      # becoming the controlling terminal.
+      SECURE_OPEN_FLAGS = ::File::NOFOLLOW | ::File::NONBLOCK | ::File::NOCTTY
+      # Seconds to wait before opening a refused log file again.
+      REFUSED_RETRY_INTERVAL = 5
+      private_constant :SECURE_OPEN_FLAGS, :REFUSED_RETRY_INTERVAL
+
       # Create an appender to log to a named file.
       #
       # Parameters
@@ -77,7 +86,12 @@ module SemanticLogger
       #     Octal file permissions to apply to the log file.
       #     Log files frequently contain sensitive information, so restrict access
       #     by supplying for example `0o640` (owner read/write, group read).
-      #     Applied both when the file is created and to an existing log file.
+      #     Applied both when the file is created and to an existing log file, which must be
+      #     a regular file owned by the process, with a single hard link, that the file_name
+      #     names directly rather than through a symbolic link. Nothing is written to any
+      #     other file, or when the file cannot be opened: the appender raises IOError when it
+      #     is created, and when the file is later reopened, it reports why and writes nothing
+      #     until the file is fixed, trying again every few seconds.
       #     Default: nil (use the process umask, the standard Ruby behavior)
       #
       #   :retry_count [Integer]
@@ -153,45 +167,34 @@ module SemanticLogger
 
         # Open the file at creation time so that misconfiguration (bad directory,
         # insufficient permissions, ...) raises immediately in SemanticLogger.add_appender,
-        # instead of surfacing later on the appender thread via the internal logger.
-        reopen
+        # instead of surfacing later on the appender thread via the internal logger. With
+        # permissions, a refused log file raises IOError (see #open_with_permissions).
+        raise(IOError, refusal_message, cause: @refusal_cause) unless reopen
       end
 
       # After forking an active process call #reopen to re-open
       # open the file handles etc to resources.
       def reopen
-        begin
-          @file&.close
-        rescue StandardError
-          nil
-        end
+        close_quietly(@file)
 
         self.current_file_name = apply_format_directives(file_name)
-        if ::File.directory?(file_name)
+        # With permissions, a directory is refused when it fails to open, like any other file.
+        if !permissions && ::File.directory?(current_file_name)
           raise(ArgumentError, "The supplied log file_name: #{current_file_name} is already a directory.")
         end
 
         self.log_count = 0
-        if append && reopen_size && ::File.exist?(current_file_name)
-          self.log_size = ::File.size(current_file_name)
-          self.log_size = 0 if log_size >= reopen_size
-        else
-          self.log_size = 0
-        end
+        # File.size? rather than File.size, since the file can be removed at any time.
+        size = ::File.size?(current_file_name) if append && reopen_size
+        self.log_size = size && size < reopen_size ? size : 0
 
         self.reopen_at = reopen_period ? next_reopen_period(reopen_period) : nil
 
         options = ::File::WRONLY | ::File::CREAT
         options |= ::File::APPEND if append
-        @file =
-          if permissions
-            ::File.open(current_file_name, options, permissions)
-          else
-            ::File.open(current_file_name, options)
-          end
-        # File.open only applies the permissions when creating the file, so also
-        # enforce them on an already existing log file.
-        @file.chmod(permissions) if permissions
+        @file = permissions ? open_with_permissions(options) : ::File.open(current_file_name, options)
+        return unless @file
+
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
         @file.sync = true
@@ -203,18 +206,23 @@ module SemanticLogger
       # Since only one appender thread will be writing to the file at a time
       # it is not necessary to protect access to the file with a semaphore.
       def log(log)
-        reopen if time_to_reopen?
+        # Read once, since #reopen can be called from another thread meanwhile.
+        file = time_to_reopen? ? reopen : @file
+        return refused unless file
 
-        count = 0
+        # Formatted once, outside the retry below: reopening the file cannot fix the content.
+        message = format_log(log) << "\n"
+        count   = 0
         begin
-          message = formatter.call(log, self) << "\n"
-          @file.write(message)
+          file.write(message)
           @log_count += 1
           @log_size  += message.size
         rescue StandardError => e
           if count < retry_count
             count += 1
-            reopen
+            file = reopen
+            return refused unless file
+
             retry
           end
           raise(e)
@@ -228,18 +236,118 @@ module SemanticLogger
         @file&.flush
       end
 
+      # See Subscriber#inspect.
+      def inspect_details
+        "file_name=#{file_name.inspect}"
+      end
+
       private
 
       attr_writer :log_count, :log_size, :current_file_name, :reopen_at
 
+      # Opens the log file and applies the permissions to it, when it is safe to do so (see
+      # #permissions_refusal). Returns the open file, or nil after closing it and recording why it is
+      # refused, so that nothing is written to it.
+      #
+      # Changing the permissions of a file that another name links to could loosen them on an
+      # unrelated file, such as a private key linked into a shared log directory. Writing log messages
+      # to a file that this process does not own, or that another name links to, would let whoever
+      # controls it read them, despite the permissions asked for. So with permissions, the appender
+      # fails closed: it raises when it is created (see #initialize), and when the file is later
+      # reopened, on a rotation, a fork, or a write retry, it stops writing and reports why (see
+      # #refused) until the file is fixed (see #refused_retry_due?).
+      def open_with_permissions(options)
+        file   = ::File.open(current_file_name, options | SECURE_OPEN_FLAGS, permissions)
+        reason = permissions_refusal(file)
+        if reason
+          close_quietly(file)
+          return refuse(reason)
+        end
+
+        file.chmod(permissions)
+        @refusal = @refusal_cause = @reported_refusal = nil
+        file
+      rescue SystemCallError => e
+        close_quietly(file)
+        refuse(open_refusal(e), e)
+      end
+
+      # Records why the log file is refused, and returns nil.
+      def refuse(reason, cause = nil)
+        @refusal          = reason
+        @refusal_cause    = cause
+        @refused_retry_at = monotonic_time + REFUSED_RETRY_INTERVAL
+        nil
+      end
+
+      # Reports why log messages are not being written to the log file, once for each file and
+      # reason, and returns false.
+      def refused
+        refusal = [current_file_name, @refusal]
+        unless @reported_refusal == refusal
+          @reported_refusal = refusal
+          logger.error(refusal_message)
+        end
+        false
+      end
+
+      def refusal_message
+        "Not writing to the log file #{current_file_name}: #{@refusal}"
+      end
+
+      # Returns why the log file could not be opened with permissions.
+      def open_refusal(error)
+        return "it is a symbolic link" if error.is_a?(Errno::ELOOP) && ::File.symlink?(current_file_name)
+
+        "it could not be opened (#{error.message})"
+      end
+
+      # Returns why the permissions must not be applied to the open log file, nor log messages
+      # written to it, or nil when they can.
+      #
+      # It must be a regular file owned by this process, with a single hard link, since a hard link
+      # planted in the log directory is otherwise indistinguishable from the file it links to. The
+      # path must also still name it, so that swapping the path after opening cannot redirect the
+      # log messages.
+      def permissions_refusal(file)
+        stat      = file.stat
+        path_stat = ::File.lstat(current_file_name)
+        return "it is not a regular file" unless stat.file?
+        return "it is not owned by this process" unless stat.uid == ::Process.euid
+        return "it has more than one hard link" unless stat.nlink == 1
+        return "it changed after it was opened" unless path_stat.dev == stat.dev && path_stat.ino == stat.ino
+
+        nil
+      rescue SystemCallError => e
+        "it could not be checked (#{e.message})"
+      end
+
+      def close_quietly(file)
+        file&.close
+      rescue StandardError
+        nil
+      end
+
+      def monotonic_time
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
       def time_to_reopen?
-        return true unless @file
+        return refused_retry_due? unless @file
 
         (reopen_count.positive? && (log_count >= reopen_count)) ||
           (reopen_size.positive? && (log_size >= reopen_size)) ||
           (reopen_at && (Time.now > reopen_at))
       end
 
+      # A refused log file is opened again every few seconds, and at the next scheduled reopen,
+      # rather than for every message.
+      def refused_retry_due?
+        return true unless @refused_retry_at
+
+        monotonic_time >= @refused_retry_at || (reopen_at && Time.now > reopen_at)
+      end
+
       def apply_format_directives(file_name)
         return file_name unless file_name.include?("%")
 

data/lib/semantic_logger/appender/graylog.rb

@@ -100,7 +100,7 @@ module SemanticLogger
 
       # Re-open after process fork
       def reopen
-        uri       = URI.parse(@url)
+        uri       = Utils.parse_url(@url)
         @server   = uri.host
         @port     = uri.port
         @protocol = uri.scheme.to_sym
@@ -120,10 +120,10 @@ module SemanticLogger
       def call(log, logger)
         h = default_formatter.call(log, logger)
 
-        h[:short_message] = h.delete(:message)
-        if h[:short_message].nil?
-          h[:short_message] = log.exception.nil? ? "<no-exception-message>" : log.exception.message
-        end
+        # GELF rejects an empty short_message, such as a message of only ANSI escape codes.
+        h[:short_message] = h.delete(:message).to_s
+        h[:short_message] = log.exception.message.to_s.strip if h[:short_message].empty? && log.exception
+        h[:short_message] = "<no-exception-message>" if h[:short_message].empty?
         h[:level]        = logger.level_map[log.level]
         h[:level_str]    = log.level.to_s
         h[:duration_str] = h.delete(:duration)
@@ -131,11 +131,27 @@ module SemanticLogger
       end
 
       # Forward log messages
+      #
+      # GELF serializes the Hash with #to_json, which raises for Infinity, NaN, invalid UTF-8, or a
+      # structure nested too deeply, and writes a Struct through its #inspect, so such a Hash is
+      # handed over as a copy that JSON can represent (see Utils.json_ready).
       def log(log)
-        notifier.notify!(formatter.call(log, self))
+        notifier.notify!(format_log(log) { |hash| Utils.json_ready(hash) })
         true
       end
 
+      # GELF raises ArgumentError for a log message it cannot send, such as one too large to fit in
+      # the maximum number of chunks, which retrying cannot send either. Unavailable servers raise
+      # network errors instead.
+      def message_error?(exception)
+        exception.is_a?(ArgumentError) || super
+      end
+
+      # The url without any credentials. See Subscriber#inspect.
+      def inspect_details
+        "url=#{Utils.redacted_url(url).inspect}"
+      end
+
       private
 
       def default_formatter

data/lib/semantic_logger/appender/honeybadger.rb

@@ -43,7 +43,7 @@ module SemanticLogger
 
       # Send an error notification to honeybadger
       def log(log)
-        context = formatter.call(log, self)
+        context = format_log(log)
         if log.exception
           context.delete(:exception)
           ::Honeybadger.notify(log.exception, context)

data/lib/semantic_logger/appender/honeybadger_insights.rb

@@ -43,7 +43,7 @@ module SemanticLogger
 
       # Send log to honeybadger events API
       def log(log)
-        event = formatter.call(log, self).merge(event_type: "log")
+        event = format_log(log).merge(event_type: "log")
 
         ::Honeybadger.event(event)