semantic_logger 5.1.0 → 5.2.0

data/lib/semantic_logger/appender/http.rb

@@ -31,7 +31,13 @@ module SemanticLogger
     class Http < SemanticLogger::Subscriber
       attr_accessor :username, :compress, :header,
                     :open_timeout, :read_timeout, :continue_timeout
-      attr_reader :http, :url, :server, :port, :path, :ssl_options, :proxy_url
+      attr_reader :http, :url, :server, :port, :path, :query, :ssl_options, :proxy_url
+
+      # Maximum number of bytes of an error response body to write to the internal logger.
+      MAX_LOGGED_RESPONSE_BODY = 512
+
+      # Describes a valid url in the error raised for an invalid one.
+      URL_FORMAT_HINT = "Should be similar to: 'http://hostname:port/path'".freeze
 
       # Create HTTP(S) log appender
       #
@@ -42,6 +48,9 @@ module SemanticLogger
       #     To enable SSL include https in the URL.
       #       Example: https://example.com/some_path
       #       verify_mode will default: OpenSSL::SSL::VERIFY_PEER
+      #     A query string is kept and sent with every request that submits log data,
+      #     for the log servers that are configured through it.
+      #       Example: http://example.com/some_path?source=my_app
       #
       #   application: [String]
       #     Name of this application to appear in log messages.
@@ -136,11 +145,10 @@ module SemanticLogger
         }.merge(header)
         @header["Content-Encoding"] = "gzip" if @compress
 
-        uri     = URI.parse(@url)
+        uri     = parse_url(@url)
         @server = uri.host
-        unless @server
-          raise(ArgumentError, "Invalid format for :url: #{@url.inspect}. Should be similar to: 'http://hostname:port/path'")
-        end
+        # The url is left out of the message, since it can hold a user and password.
+        raise(ArgumentError, "Invalid format for :url, it has no host. #{URL_FORMAT_HINT}") unless @server
 
         @port     = uri.port
         @username = uri.user if !@username && uri.user
@@ -148,6 +156,11 @@ module SemanticLogger
         @path     = uri.path
         # Path cannot be empty
         @path     = "/" if @path == ""
+        # Query string of the url, applied to every request, see #uri_with_query.
+        # Kept separate from the path, because subclasses build their own request paths
+        # from it. For example Appender::ElasticsearchHttp appends the index name.
+        @query    = uri.query
+        @query    = nil if @query == ""
 
         if uri.scheme == "https"
           @ssl_options[:use_ssl] = true
@@ -157,7 +170,7 @@ module SemanticLogger
           @port ||= HTTP.http_default_port
         end
 
-        @proxy_uri = URI.parse(@proxy_url) if @proxy_url && @proxy_url != :ENV
+        @proxy_uri = Utils.parse_url(@proxy_url, option: ":proxy_url") if @proxy_url && @proxy_url != :ENV
 
         @http = nil
 
@@ -196,7 +209,7 @@ module SemanticLogger
 
       # Forward log messages to HTTP Server
       def log(log)
-        message = formatter.call(log, self)
+        message = format_log(log)
         logger.trace(message)
         post(message)
       end
@@ -204,7 +217,7 @@ module SemanticLogger
       # Forward a batch of log messages to the HTTP Server in a single request.
       # Only used when the appender is created with `batch: true`.
       def batch(logs)
-        message = formatter.batch(logs, self)
+        message = format_batch(logs)
         logger.trace(message)
         post(message)
       end
@@ -216,8 +229,30 @@ module SemanticLogger
         false
       end
 
+      # The whole batch is formatted into a single request before anything is sent.
+      def batch_all_or_nothing?
+        true
+      end
+
+      # Returns [String] the scheme, host, and any non-default port of the url. Safe to log.
+      # See Utils.redacted_url.
+      def redacted_url
+        @redacted_url ||= Utils.redacted_url(url)
+      end
+
+      # Only the redacted url, since the instance variables hold the password, the authorization
+      # headers, and any credential a subclass adds. See Subscriber#inspect.
+      def inspect_details
+        "url=#{redacted_url.inspect}"
+      end
+
       private
 
+      # Returns [URI] the parsed url, without the url in the error raised for an invalid one.
+      def parse_url(url)
+        Utils.parse_url(url, hint: URL_FORMAT_HINT)
+      end
+
       # Use JSON Formatter by default
       def default_formatter
         SemanticLogger::Formatters::Json.new
@@ -233,22 +268,43 @@ module SemanticLogger
 
       # HTTP Post
       def post(body, request_uri = path)
-        request = Net::HTTP::Post.new(request_uri, @header)
+        request = Net::HTTP::Post.new(uri_with_query(request_uri), @header)
         process_request(request, body)
       end
 
       # HTTP Put
       def put(body, request_uri = path)
-        request = Net::HTTP::Put.new(request_uri, @header)
+        request = Net::HTTP::Put.new(uri_with_query(request_uri), @header)
         process_request(request, body)
       end
 
       # HTTP Delete
+      #
+      # The query string of the configured url is _not_ applied here: it configures how
+      # the server ingests log data, and servers reject it on a maintenance request.
+      # Appender::ElasticsearchHttp#delete_all deletes an index, and Elasticsearch fails
+      # the request with a 400 when it carries unrecognized parameters.
       def delete(request_uri = path)
         request = Net::HTTP::Delete.new(request_uri, @header)
         process_request(request)
       end
 
+      # Applies the query string of the configured url, if any, to the uri of a request
+      # that submits log data.
+      #
+      # Kept out of `path` on purpose: subclasses build their own request paths from it
+      # (Appender::ElasticsearchHttp appends the index name and `_doc` to it), and the
+      # query has to stay at the end of the request uri regardless.
+      #
+      # @param request_uri [String] path of the request, `path` by default.
+      # @return [String]
+      def uri_with_query(request_uri)
+        return request_uri unless @query
+
+        separator = request_uri.include?("?") ? "&" : "?"
+        "#{request_uri}#{separator}#{@query}"
+      end
+
       # Process HTTP Request
       def process_request(request, body = nil)
         if body
@@ -260,7 +316,10 @@ module SemanticLogger
           true
         else
           # Failures are logged to the global semantic logger failsafe logger (Usually stderr or file)
-          logger.error("Bad HTTP response from: #{url} code: #{response.code}, #{response.body}")
+          logger.error(
+            "Bad HTTP response from: #{redacted_url} code: #{response.code}, " \
+            "#{response.body.to_s[0, MAX_LOGGED_RESPONSE_BODY].inspect}"
+          )
           false
         end
       rescue RuntimeError => e

data/lib/semantic_logger/appender/io.rb

@@ -50,7 +50,7 @@ module SemanticLogger
         # Since only one appender thread will be writing to the file at a time
         # it is not necessary to protect access to the file with a semaphore
         # Allow this logger to filter out log levels lower than it's own
-        @io.write(formatter.call(log, self) << "\n")
+        @io.write(format_log(log) << "\n")
         true
       end
 

data/lib/semantic_logger/appender/kafka.rb

@@ -23,6 +23,9 @@ require "date"
 module SemanticLogger
   module Appender
     class Kafka < SemanticLogger::Subscriber
+      SEED_BROKERS_HINT = "Each should be similar to: 'hostname:port' or 'kafka://hostname:port'".freeze
+      private_constant :SEED_BROKERS_HINT
+
       attr_accessor :seed_brokers, :client_id, :connect_timeout, :socket_timeout,
                     :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key, :ssl_ca_certs_from_system,
                     :delivery_threshold, :delivery_interval, :required_acks,
@@ -149,17 +152,22 @@ module SemanticLogger
       end
 
       def reopen
-        @kafka = ::Kafka.new(
-          seed_brokers:             seed_brokers,
-          client_id:                client_id,
-          connect_timeout:          connect_timeout,
-          socket_timeout:           socket_timeout,
-          ssl_ca_cert:              ssl_ca_cert,
-          ssl_client_cert:          ssl_client_cert,
-          ssl_client_cert_key:      ssl_client_cert_key,
-          ssl_ca_certs_from_system: ssl_ca_certs_from_system,
-          logger:                   logger
-        )
+        # ruby-kafka puts the seed broker, which can hold credentials, in the error it raises for an
+        # invalid one: URI::InvalidURIError, or Kafka::Error for an unknown scheme.
+        errors = [URI::InvalidURIError, ::Kafka::Error]
+        @kafka = Utils.without_url_in_errors(option: ":seed_brokers", hint: SEED_BROKERS_HINT, errors: errors) do
+          ::Kafka.new(
+            seed_brokers:             seed_brokers,
+            client_id:                client_id,
+            connect_timeout:          connect_timeout,
+            socket_timeout:           socket_timeout,
+            ssl_ca_cert:              ssl_ca_cert,
+            ssl_client_cert:          ssl_client_cert,
+            ssl_client_cert_key:      ssl_client_cert_key,
+            ssl_ca_certs_from_system: ssl_ca_certs_from_system,
+            logger:                   logger
+          )
+        end
 
         @producer = @kafka.async_producer(
           delivery_threshold: delivery_threshold,
@@ -177,7 +185,7 @@ module SemanticLogger
 
       # Forward log messages to Kafka producer thread.
       def log(log)
-        json = formatter.call(log, self)
+        json = format_log(log)
         @producer.produce(json, topic: topic, partition: partition, partition_key: partition_key, key: key)
       end
 
@@ -195,6 +203,12 @@ module SemanticLogger
         )
       end
 
+      # Only the topic: the seed brokers can hold credentials, and the options hold the TLS private
+      # key and SASL passwords. See Subscriber#inspect.
+      def inspect_details
+        "topic=#{@topic.inspect}"
+      end
+
       private
 
       attr_reader :producer

data/lib/semantic_logger/appender/loki.rb

@@ -33,6 +33,8 @@ module SemanticLogger
       #   url: [String]
       #     Define the loki instance URL.
       #     Example: https://logs-prod-999.grafana.net
+      #     A query string is kept and sent with every request.
+      #     Example: https://logs-prod-999.grafana.net?orgid=my_org
       #     Default: nil
       def initialize(url: nil,
                      formatter: SemanticLogger::Formatters::Loki.new,
@@ -40,18 +42,26 @@ module SemanticLogger
                      path: INGESTION_PATH,
                      **args,
                      &)
-        super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &)
+        super(url: self.class.ingestion_url(url, path), formatter: formatter, header: header, **args, &)
+      end
+
+      # Appends the ingestion path to the supplied url, keeping any query string at the
+      # end of the url where Appender::Http can parse it out.
+      def self.ingestion_url(url, path)
+        uri      = Utils.parse_url(url.to_s)
+        uri.path = "#{uri.path.chomp('/')}/#{path.delete_prefix('/')}"
+        uri.to_s
       end
 
       def log(log)
-        message = formatter.call(log, self)
+        message = format_log(log)
         logger.trace(message)
         post(message)
       end
 
       # Logs in batches
       def batch(logs)
-        message = formatter.batch(logs, self)
+        message = format_batch(logs)
         logger.trace(message)
         post(message)
       end

data/lib/semantic_logger/appender/mongodb.rb

@@ -164,10 +164,29 @@ module SemanticLogger
       # Log the message to MongoDB
       def log(log)
         # Insert log entry into Mongo
-        collection.insert_one(formatter.call(log, self))
+        collection.insert_one(format_log(log))
         true
       end
 
+      # Errors raised for a document that cannot be stored, which retrying cannot insert either.
+      # BSON raises BSON::Error, ArgumentError (a key with a NUL byte), or RangeError (an Integer too
+      # large for 64 bits) while the document is serialized, before anything is sent, and the driver
+      # raises Mongo::Error::MaxBSONSize for a document larger than the server accepts.
+      DOCUMENT_ERRORS = [BSON::Error, ArgumentError, RangeError, Mongo::Error::MaxBSONSize].freeze
+
+      # Except BSON::Error::BSONDecodeError, which comes from reading the server's reply rather than
+      # from the document, so it is retried like any other error from the server.
+      def message_error?(exception)
+        return false if exception.is_a?(BSON::Error::BSONDecodeError)
+
+        super || DOCUMENT_ERRORS.any? { |error| exception.is_a?(error) }
+      end
+
+      # Only the collection name: the uri can hold a user and password. See Subscriber#inspect.
+      def inspect_details
+        "collection=#{@collection_name.inspect}"
+      end
+
       private
 
       def default_formatter

data/lib/semantic_logger/appender/new_relic.rb

@@ -64,7 +64,7 @@ module SemanticLogger
         # For more documentation on the NewRelic::Agent.notice_error method see:
         # http://rubydoc.info/github/newrelic/rpm/NewRelic/Agent#notice_error-instance_method
         # and https://docs.newrelic.com/docs/ruby/ruby-agent-api
-        ::NewRelic::Agent.notice_error(exception, formatter.call(log, self))
+        ::NewRelic::Agent.notice_error(exception, format_log(log))
         true
       end
     end

data/lib/semantic_logger/appender/new_relic_logs.rb

@@ -57,17 +57,15 @@ module SemanticLogger
 
       # Send an error notification to New Relic
       def log(log)
-        begin
-          message = formatter.call(log, self) # Generate the structured log
-          json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
-          level = log.level.to_s.upcase # Determine the log level
-          self.class.log_newrelic(json_message, level)
-        rescue JSON::GeneratorError => e
-          warn("Failed to serialize log message to JSON: #{e.message}")
-          warn("Problematic data: #{message.inspect}")
-        rescue StandardError => e
-          warn("Unexpected error while logging to New Relic: #{e.message}")
-        end
+        message      = format_log(log) # Generate the structured log
+        json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
+        level        = log.level.to_s.upcase # Determine the log level
+        self.class.log_newrelic(json_message, level)
+        true
+      rescue StandardError => e
+        # Reported to the internal logger: this appender's own #warn would log the error through
+        # this method again, recursing until SystemStackError.
+        logger.warn("Could not log to New Relic", e)
         true
       end
 

data/lib/semantic_logger/appender/open_telemetry.rb

@@ -48,7 +48,7 @@ module SemanticLogger
       def log(log)
         # return log_metric(log) if metrics && log.metric_only?
 
-        body        = formatter.call(log, self)
+        body        = format_log(log)
         level       = body.delete(:level)
         level_index = body.delete(:level_index)
         time        = body.delete(:time)
@@ -89,7 +89,7 @@ module SemanticLogger
       # For logging metrics only log events.
       # def log_metric(log)
       #   puts "**** TODO: Metric Only Event ****"
-      #   ap formatter.call(log, self)
+      #   ap format_log(log)
       #   ap log.payload
       #   true
       # end

data/lib/semantic_logger/appender/rabbitmq.rb

@@ -107,7 +107,7 @@ module SemanticLogger
       end
 
       def log(log)
-        queue.publish(formatter.call(log, self))
+        queue.publish(format_log(log))
       end
 
       def flush
@@ -122,6 +122,11 @@ module SemanticLogger
       def queue
         @queue ||= @channel.queue(@queue_name)
       end
+
+      # Only the queue name: the connection options hold the password. See Subscriber#inspect.
+      def inspect_details
+        "queue_name=#{@queue_name.inspect}"
+      end
     end
   end
 end

data/lib/semantic_logger/appender/sentry.rb

@@ -54,7 +54,7 @@ module SemanticLogger
         # Ignore logs coming from Raven itself
         return false if log.name == "Raven"
 
-        context      = formatter.call(log, self)
+        context      = format_log(log)
         user         = context.delete(:user)
         tags         = context.delete(:tags)
         attrs        = {

data/lib/semantic_logger/appender/sentry_ruby.rb

@@ -12,6 +12,15 @@ end
 module SemanticLogger
   module Appender
     class SentryRuby < SemanticLogger::Subscriber
+      # Keys of the named tags or payload that set the Sentry user, mapped to its fields.
+      USER_KEYS = {user_id: :id, username: :username, user_email: :email, ip_address: :ip_address}.freeze
+
+      # The types of value taken from the payload for a field of the Sentry user.
+      USER_VALUE_TYPES = [String, Symbol, Integer].freeze
+
+      # The types of value accepted in a fingerprint, which Sentry turns into Strings.
+      FINGERPRINT_PART_TYPES = [String, Symbol, Numeric].freeze
+
       # Create Appender
       #
       # Parameters
@@ -54,16 +63,19 @@ module SemanticLogger
         # Ignore logs coming from Sentry itself
         return false if log.name == "Sentry"
 
-        context = formatter.call(log, self)
-        payload = context.delete(:payload) || {}
-        named_tags = context[:named_tags] || {}
-        transaction_name = named_tags.delete(:transaction_name)
+        context = format_log(log)
+        # Copies, since the payload and named tags are the log entry's own, which every other
+        # appender writes after this one, and the payload can be the application's own Hash. The
+        # keys taken out below must only be left out of what is sent to Sentry.
+        payload               = payload_copy(context.delete(:payload))
+        named_tags            = (context[:named_tags] || {}).dup
+        context[:named_tags]  = named_tags
+        transaction_name      = named_tags.delete(:transaction_name)
+        fingerprint           = extract_fingerprint!(named_tags) || extract_fingerprint!(payload, from_payload: true)
 
         user = extract_user!(named_tags, payload)
         tags = extract_tags!(context)
 
-        fingerprint = payload.delete(:fingerprint)
-
         ::Sentry.with_scope do |scope|
           scope.set_user(user) if user
           scope.set_level(context.delete(:level)) if context[:level]
@@ -100,26 +112,88 @@ module SemanticLogger
       # Any additional value nested in a :user key will be added, provided any of
       # the above keys is already present.
       #
-      def extract_user!(*sources)
-        keys = {user_id: :id, username: :username, user_email: :email, ip_address: :ip_address}
-
+      # The payload can hold untrusted data, such as a request body, so taking the user from it is
+      # deprecated, and the named tags win: a payload field is only taken when the named tags do not
+      # set it, and its value is a String, Symbol, or Integer. A payload value that is not taken is
+      # left where it is, and sent to Sentry as data.
+      def extract_user!(named_tags, payload)
         user = {}
 
-        sources.each do |source|
-          keys.each do |source_key, target_key|
-            value = source.delete(source_key)
-            user[target_key] = value if value
+        USER_KEYS.each do |source_key, target_key|
+          value = named_tags.delete(source_key)
+          if value
+            user[target_key] = value
+          elsif user_value?(payload[source_key])
+            user[target_key] = payload.delete(source_key)
+            payload_context_deprecated
           end
         end
 
         return if user.empty?
 
-        sources.each do |source|
-          extras = source.delete(:user)
-          user.merge!(extras) if extras.is_a?(Hash)
+        user.merge!(named_tags.delete(:user)) if named_tags[:user].is_a?(Hash)
+        merge_payload_user!(user, payload)
+        user
+      end
+
+      # Adds the fields of a :user Hash in the payload that the user does not have yet, when each of
+      # its keys is a Symbol or String, and each of its values a String, Symbol, or Integer.
+      def merge_payload_user!(user, payload)
+        extras = payload[:user]
+        return unless extras.is_a?(Hash) && extras.all? { |key, value| user_key?(key) && user_value?(value) }
+
+        payload.delete(:user)
+        payload_context_deprecated
+        extras.each do |key, value|
+          user[key.to_sym] = value unless user.key?(key.to_sym) || user.key?(key.to_s)
         end
+      end
 
-        user
+      def user_key?(key)
+        key.is_a?(Symbol) || key.is_a?(String)
+      end
+
+      def user_value?(value)
+        USER_VALUE_TYPES.any? { |type| value.is_a?(type) }
+      end
+
+      # Returns [Array<String>] the fingerprint taken from the named tags or payload, an Array of
+      # Strings, Symbols, or numbers. Returns nil when there is none, or it is of another type, which
+      # is then left where it is and sent to Sentry as data. As for the user, taking it from the
+      # payload is deprecated.
+      def extract_fingerprint!(source, from_payload: false)
+        fingerprint = source[:fingerprint]
+        return unless fingerprint.is_a?(Array) && fingerprint.all? { |part| fingerprint_part?(part) }
+
+        payload_context_deprecated if from_payload
+        source.delete(:fingerprint).map(&:to_s)
+      end
+
+      def fingerprint_part?(part)
+        FINGERPRINT_PART_TYPES.any? { |type| part.is_a?(type) }
+      end
+
+      # Returns [Hash] a copy of the payload that keys can be taken out of. A payload that is not a
+      # Hash is sent to Sentry as the extra named payload.
+      def payload_copy(payload)
+        case payload
+        when nil  then {}
+        when Hash then payload.dup
+        else {payload: payload}
+        end
+      end
+
+      # Warns once that the Sentry user or fingerprint was taken from the payload.
+      def payload_context_deprecated
+        return if @payload_context_deprecated
+
+        @payload_context_deprecated = true
+        Kernel.warn(
+          "SemanticLogger::Appender::SentryRuby took the Sentry user or fingerprint from the log payload. " \
+          "This is deprecated and will be removed in v6, since the payload can hold untrusted data: " \
+          "set them with named tags instead, for example SemanticLogger.tagged(user_id: user.id).",
+          category: :deprecated
+        )
       end
 
       # Extract tags.

data/lib/semantic_logger/appender/splunk.rb

@@ -9,6 +9,10 @@ end
 #
 # Use the official splunk gem to log messages to Splunk.
 #
+# Deprecated: will be removed in v6. Use the :splunk_http appender instead.
+# The splunk-sdk-ruby gem (last released in 2015) does not verify the server's TLS certificate,
+# so its connection, including the login credentials, is open to interception.
+#
 # Example
 #   SemanticLogger.add_appender(
 #     appender: :splunk,
@@ -90,6 +94,11 @@ module SemanticLogger
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
       def initialize(index: "main", source_type: nil, **args, &)
+        Kernel.warn(
+          "SemanticLogger::Appender::Splunk (appender: :splunk) is deprecated and will be removed in v6. " \
+          "Its splunk-sdk-ruby gem does not verify TLS certificates. Use appender: :splunk_http instead.",
+          category: :deprecated
+        )
         @index       = index
         @source_type = source_type
 
@@ -109,7 +118,7 @@ module SemanticLogger
 
       # Log the message to Splunk
       def log(log)
-        event = formatter.call(log, self)
+        event = format_log(log)
         service_index.submit(event.delete(:message), event)
         true
       end