logstash-logger 0.26.1 → 1.1.0

data/lib/logstash-logger/device/http.rb

@@ -0,0 +1,104 @@
+require 'uri'
+require 'net/http'
+
+module LogStashLogger
+  module Device
+
+    # Rudimentary write to Logstash HTTP Input relying on buffering
+    # rather than persistent HTTP connections for efficiency.
+    class HTTP < Connectable
+
+      # Request body formats, with the Content-Type sent by default for each.
+      FORMATS = {
+        json_array: "application/json",
+        ndjson: "application/x-ndjson",
+      }.freeze
+
+      DEFAULT_FORMAT = :json_array
+
+      # Statuses that reject the content of the request. Sending the same
+      # records again can never succeed, so the rejected records are found and
+      # dropped. Every other error status is retried.
+      CONTENT_REJECTED_STATUSES = [
+        400, # Bad Request: the body is malformed
+        413, # Content Too Large: the body is over the collector's size limit
+        415, # Unsupported Media Type: the collector does not accept this content type
+        422, # Unprocessable Content: the body is well-formed but cannot be processed
+      ].freeze
+
+      def initialize(opts)
+        super
+        @url = URI(opts[:url])
+        @format = (opts[:format] || DEFAULT_FORMAT).to_sym
+        unless FORMATS.key?(@format)
+          raise ArgumentError, "Unknown HTTP body format: #{@format.inspect} (expected one of #{FORMATS.keys.map(&:inspect).join(', ')})"
+        end
+        @content_type = opts[:content_type] || FORMATS[@format]
+      end
+
+      def connect
+        # no-op
+      end
+
+      def write_one(message)
+        write_batch([message])
+      end
+
+      def write_batch(messages, _group = nil)
+        messages = messages.map(&:to_s)
+        resp = post(messages)
+        return if resp.is_a?(Net::HTTPSuccess)
+        raise upload_error(resp) unless content_rejected?(resp)
+        deliver_one_at_a_time(messages, resp)
+      end
+
+      private
+
+      def post(messages)
+        body = @format == :ndjson ? ndjson_body(messages) : json_array_body(messages)
+        Net::HTTP.post @url, body, {"Content-Type" => @content_type}
+      end
+
+      # The collector rejected the content of the batch. Send the records one
+      # at a time and drop only the ones that are still rejected, so one bad
+      # record does not hold back, or take down, the rest.
+      def deliver_one_at_a_time(messages, batch_resp)
+        return drop(messages.first, batch_resp) if messages.size == 1
+
+        messages.each do |message|
+          resp = post([message])
+          next if resp.is_a?(Net::HTTPSuccess)
+
+          if content_rejected?(resp)
+            drop(message, resp)
+          else
+            raise upload_error(resp)
+          end
+        end
+      end
+
+      def content_rejected?(resp)
+        CONTENT_REJECTED_STATUSES.include?(resp.code.to_i)
+      end
+
+      def drop(message, resp)
+        log_warning("dropping a record of #{message.bytesize} bytes that the collector rejected: HTTP #{resp.code} #{resp.message}")
+      end
+
+      def upload_error(resp)
+        RuntimeError.new("HTTP log upload failed: #{resp.code} #{resp.message}").tap { |e| log_error(e) }
+      end
+
+      # Logstash HTTP input expects a JSON array instead of lines of JSON
+      def json_array_body(messages)
+        "[#{messages.join(',')}]"
+      end
+
+      # Newline-delimited JSON: one event per line, each ending with a newline
+      def ndjson_body(messages)
+        messages.map { |m| m.end_with?("\n") ? m : "#{m}\n" }.join
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/kafka.rb

@@ -1,56 +1,200 @@
-require 'poseidon'
-
 module LogStashLogger
   module Device
     class Kafka < Connectable
+      class TLSConfiguration
+        attr_reader :ssl_ca_cert, :ssl_client_cert, :ssl_client_cert_key
+
+        def initialize(opts = {})
+          @ssl_ca_cert = opts[:ssl_ca_cert]
+          @ssl_client_cert = opts[:ssl_client_cert]
+          @ssl_client_cert_key = opts[:ssl_client_cert_key]
+        end
+
+        def cert_bundle
+          @cert_bundle ||= all_cert_params? ? cert_params_as_hash : {}
+        end
+
+        def valid?
+          all_cert_params? || no_cert_params?
+        end
+
+        def invalid?
+          !valid?
+        end
+
+        private
+
+        def cert_params_as_hash
+          { ssl_ca_cert: @ssl_ca_cert,
+            ssl_client_cert: @ssl_client_cert,
+            ssl_client_cert_key: @ssl_client_cert_key,
+          }
+        end
+
+        def all_cert_params?
+          cert_params_as_hash.values.compact.length == valid_cert_params_length
+        end
+
+        def no_cert_params?
+          cert_params_as_hash.values.compact.empty?
+        end
+
+        def valid_cert_params_length
+          cert_params_as_hash.keys.length
+        end
+      end
 
-      DEFAULT_HOST = 'localhost'
-      DEFAULT_PORT = 9092
-      DEFAULT_TOPIC = 'logstash'
-      DEFAULT_PRODUCER = 'logstash-logger'
-      DEFAULT_BACKOFF = 1
-
-      attr_accessor :hosts, :topic, :producer, :backoff
-
-      def initialize(opts)
-        super
-        host = opts[:host] || DEFAULT_HOST
-        port = opts[:port] || DEFAULT_PORT
-        @hosts = opts[:hosts] || host.split(',').map { |h| "#{h}:#{port}" }
-        @topic = opts[:path] || DEFAULT_TOPIC
-        @producer = opts[:producer] || DEFAULT_PRODUCER
-        @backoff = opts[:backoff] || DEFAULT_BACKOFF
+      attr_reader :topic, :brokers, :cert_bundle, :kafka_tls_configurator,
+        :client_id
+
+      # Errors that will recur on every retry of the same records. Looked up by
+      # name at rescue time, because ruby-kafka is only required in #initialize.
+      NOT_RETRYABLE_ERRORS = %w[MessageSizeTooLarge RecordListTooLarge BufferOverflow].freeze
+
+      def initialize(opts = {}, kafka_tls_configurator = TLSConfiguration)
+        require 'ruby-kafka'
+        super(opts)
+
+        opts = normalize_uri_options(opts)
+        @client_id = opts[:client_id]
+        @topic = normalize_topic(opts[:topic])
         @buffer_group = @topic
+        @kafka_tls_configurator = kafka_tls_configurator
+        @brokers = make_brokers_array(opts[:brokers])
+        raise_no_brokers_set! if @brokers.empty?
+        make_cert_bundle(opts)
+      end
+
+      def connection
+        @io ||= ::Kafka.new(**kafka_client_connection_hash)
       end
 
       def connect
-        @io = ::Poseidon::Producer.new(@hosts, @producer)
-      end
-
-      def with_connection
-        connect unless connected?
-        yield
-      rescue ::Poseidon::Errors::ChecksumError, Poseidon::Errors::UnableToFetchMetadata => e
-        log_error(e)
-        log_warning("reconnect/retry")
-        sleep backoff if backoff
-        reconnect
-        retry
-      rescue => e
-        log_error(e)
-        log_warning("giving up")
-        close(flush: false)
+        @io = connection
       end
 
-      def write_batch(messages, topic = nil)
+      def write_one(message, topic=nil)
         topic ||= @topic
+        write_messages_to_broker_and_deliver do |producer|
+          producer.produce(message, topic: topic)
+        end
+      end
+
+      def write_batch(messages, topic=nil)
+        topic ||= @topic
+        write_messages_to_broker_and_deliver do |producer|
+          messages.each {|msg| producer.produce(msg, topic: topic) }
+        end
+      rescue => e
+        raise if retryable_error?(e)
+        deliver_one_at_a_time(messages, topic)
+      end
+
+      private
+
+      # One record the broker or client will never accept fails the whole batch.
+      # Send the records one at a time and drop only the ones that are still
+      # rejected. Retryable errors raise, so the buffer retries the batch.
+      def deliver_one_at_a_time(messages, topic)
+        messages.each do |message|
+          begin
+            write_one(message, topic)
+          rescue => e
+            raise if retryable_error?(e)
+            log_warning("dropping a record of #{message.to_s.bytesize} bytes that Kafka rejected: #{e.class}")
+          end
+        end
+      end
+
+      def retryable_error?(error)
+        NOT_RETRYABLE_ERRORS.none? do |name|
+          ::Kafka.const_defined?(name) && error.is_a?(::Kafka.const_get(name))
+        end
+      end
+
+      def write_messages_to_broker_and_deliver(&block)
         with_connection do
-          @io.send_messages messages.map { |message| Poseidon::MessageToSend.new(topic, message) }
+          kproducer = producer
+          block.call(kproducer) if block_given?
+          kproducer.deliver_messages
+        end
+      end
+
+      def close!
+        begin
+          if @producer
+            if @producer.respond_to?(:shutdown)
+              @producer.shutdown
+            elsif @producer.respond_to?(:close)
+              @producer.close
+            end
+          end
+        ensure
+          @producer = nil
+          super
+        end
+      end
+
+      def kafka_client_connection_hash
+        { seed_brokers: @brokers,
+          client_id: @client_id,
+        }.merge(@cert_bundle)
+      end
+
+      def producer
+        @producer ||= (connected? ? @io : connection).producer
+      end
+
+      def raise_no_topic_set!
+        fail ArgumentError, "a topic must be configured"
+      end
+
+      def raise_no_brokers_set!
+        fail ArgumentError, "brokers must be configured"
+      end
+
+      def normalize_topic(topic)
+        normalized = topic.to_s.strip
+        return raise_no_topic_set! if normalized.empty?
+        normalized
+      end
+
+      def make_brokers_array(opt)
+        brokers =
+          case opt
+          when Array
+            opt.flatten
+          when String
+            opt.split(/\s+/)
+          else
+            []
+          end
+        brokers.compact.map(&:to_s).reject(&:empty?)
+      end
+
+      def normalize_uri_options(opts)
+        normalized = opts.dup
+        if normalized[:brokers].nil?
+          if normalized[:hosts]
+            normalized[:brokers] = normalized[:hosts]
+          elsif normalized[:host]
+            broker = normalized[:host]
+            broker = "#{broker}:#{normalized[:port]}" if normalized[:port]
+            normalized[:brokers] = broker
+          end
+        end
+        if normalized[:topic].nil? && normalized[:path]
+          normalized[:topic] = normalized[:path].to_s.sub(%r{\A/}, '')
         end
+        normalized
       end
 
-      def write_one(message, topic = nil)
-        write_batch([message], topic)
+      def make_cert_bundle(opts)
+        tls_conf = kafka_tls_configurator.new(opts)
+        if tls_conf.invalid?
+          fail ArgumentError, "all ssl parameters (ssl_ca_cert, ssl_client_cert and ssl_client_cert_key) are required or do use any of them to not use TLS"
+        end
+        @cert_bundle ||= tls_conf.cert_bundle
       end
     end
   end

data/lib/logstash-logger/device/kinesis.rb

@@ -14,9 +14,18 @@ module LogStashLogger
         "ServiceUnavailable",
         "Throttling",
         "RequestExpired",
-        "ProvisionedThroughputExceededException"
+        "ProvisionedThroughputExceededException",
+        "InternalFailure"
       ].freeze
 
+      # https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+      # Records count the partition key. 1 MiB is the default and smallest
+      # per-stream maximum; raise it with aws_max_record_bytes if the stream
+      # is configured for larger records.
+      def self.max_records; 500; end
+      def self.max_record_bytes; 1_048_576; end
+      def self.max_request_bytes; 10_485_760; end
+
       def transform_message(message)
         {
           data: message,

data/lib/logstash-logger/device/redis.rb

@@ -13,6 +13,7 @@ module LogStashLogger
         @buffer_group = @list
 
         normalize_path(opts)
+        delete_unknown_keywords(opts)
 
         @redis_options = opts
       end
@@ -59,11 +60,17 @@ module LogStashLogger
       def normalize_path(opts)
         path = opts.fetch(:path, nil)
         if path
-          opts[:db] = path.gsub("/", "").to_i unless path.empty?
+          opts[:db] = path.delete("/").to_i unless path.empty?
           opts.delete(:path)
         end
       end
 
+      def delete_unknown_keywords(opts)
+        # due to restrictions in redis client version >= 5
+        # Continous error is being logged for unknown keywords, at present there is only one i.e. :sync
+        # to prevent that adding following line :)
+        opts.delete(:sync)
+      end
     end
   end
 end

data/lib/logstash-logger/device/tcp.rb

@@ -1,4 +1,5 @@
 require 'openssl'
+require 'ipaddr'
 
 module LogStashLogger
   module Device
@@ -12,16 +13,17 @@ module LogStashLogger
         @ssl_context = opts[:ssl_context]
         @use_ssl = !!(@ssl_certificate || opts[:ssl_context])
         @use_ssl = opts[:ssl_enable] if opts.has_key? :ssl_enable
-        if opts.has_key?(:use_ssl)
-          @use_ssl = opts[:use_ssl]
-          warn "[LogStashLogger] The use_ssl option is deprecated. Use ssl_enable instead."
-        end
         @verify_hostname = opts.fetch(:verify_hostname, true)
+        validate_ssl_certificate! if use_ssl? && @ssl_certificate && !@ssl_context
       end
 
+      # The TLS context used for the connection. Without an operator-supplied
+      # :ssl_context, the server certificate is always verified: against the
+      # :ssl_certificate file when one is given, otherwise against the system
+      # trust store.
       def ssl_context
         return unless use_ssl?
-        @ssl_context || certificate_context
+        @ssl_context || certificate_context || default_context
       end
 
       def use_ssl?
@@ -41,7 +43,7 @@ module LogStashLogger
           ssl_io
         else
           tcp_io
-        end
+                end
       end
 
       protected
@@ -53,20 +55,60 @@ module LogStashLogger
       end
 
       def ssl_io
-        ssl_context ?
-          OpenSSL::SSL::SSLSocket.new(tcp_io, ssl_context) :
-          OpenSSL::SSL::SSLSocket.new(tcp_io)
+        socket = tcp_io
+        begin
+          OpenSSL::SSL::SSLSocket.new(socket, ssl_context).tap do |ssl|
+            # Closing the TLS socket also closes the TCP socket underneath it
+            ssl.sync_close = true
+            # SNI is only sent for DNS names, never for IP addresses
+            ssl.hostname = hostname unless ip_address?(hostname)
+          end
+        rescue StandardError
+          socket.close rescue nil
+          raise
+        end
       end
 
+      # Trusts the certificate (or CA bundle) in the :ssl_certificate file, in
+      # addition to the system trust store.
       def certificate_context
         return unless @ssl_certificate
-        @certificate_context ||= OpenSSL::SSL::SSLContext.new.tap do |ctx|
-          ctx.set_params(cert: @ssl_certificate)
+        @certificate_context ||= verifying_context(ca_file: @ssl_certificate)
+      end
+
+      # Used when TLS is enabled without an :ssl_context or :ssl_certificate.
+      def default_context
+        @default_context ||= verifying_context
+      end
+
+      def verifying_context(params = {})
+        OpenSSL::SSL::SSLContext.new.tap do |ctx|
+          # VERIFY_PEER, hostname verification and the system certificate store
+          ctx.set_params(params)
+          # Honour an explicit verify_hostname: false
+          ctx.verify_hostname = !!@verify_hostname
+        end
+      end
+
+      def validate_ssl_certificate!
+        path = @ssl_certificate.to_s
+        unless ::File.file?(path) && ::File.readable?(path)
+          raise ArgumentError, "ssl_certificate must be a readable certificate file: #{path.inspect}"
         end
+        OpenSSL::X509::Certificate.new(::File.read(path))
+      rescue OpenSSL::X509::CertificateError => e
+        raise ArgumentError, "ssl_certificate #{path.inspect} is not a PEM or DER certificate: #{e.message}"
+      end
+
+      def ip_address?(name)
+        IPAddr.new(name.to_s)
+        true
+      rescue IPAddr::Error
+        false
       end
 
       def verify_hostname?
-        return false unless ssl_context
+        return false unless use_ssl?
         !! @verify_hostname
       end
 

data/lib/logstash-logger/device.rb

@@ -20,6 +20,7 @@ module LogStashLogger
     autoload :Stderr, 'logstash-logger/device/stderr'
     autoload :Balancer, 'logstash-logger/device/balancer'
     autoload :MultiDelegator, 'logstash-logger/device/multi_delegator'
+    autoload :HTTP, 'logstash-logger/device/http'
 
     def self.new(opts)
       opts = opts.dup
@@ -27,7 +28,7 @@ module LogStashLogger
     end
 
     def self.build_device(opts)
-      if parsed_uri_opts = parse_uri_config(opts)
+      if (parsed_uri_opts = parse_uri_config(opts))
         opts.delete(:uri)
         opts.merge!(parsed_uri_opts)
       end
@@ -38,10 +39,30 @@ module LogStashLogger
     end
 
     def self.parse_uri_config(opts)
-      if uri = opts[:uri]
+      if (uri = opts[:uri])
         require 'uri'
         parsed = ::URI.parse(uri)
         {type: parsed.scheme, host: parsed.host, port: parsed.port, path: parsed.path}
+          .merge(parse_uri_query(parsed.query))
+      end
+    end
+
+    def self.parse_uri_query(query)
+      return {} unless query && !query.empty?
+      ::URI.decode_www_form(query).each_with_object({}) do |(key, value), acc|
+        next unless key
+        key = key.to_s.strip
+        next if key.empty?
+        acc[key.to_sym] = cast_uri_value(key, value)
+      end
+    end
+
+    def self.cast_uri_value(key, value)
+      case key
+      when 'buffer_max_items'
+        value.to_i
+      else
+        value
       end
     end
 
@@ -60,6 +81,7 @@ module LogStashLogger
         when :stderr then Stderr
         when :multi_delegator then MultiDelegator
         when :balancer then Balancer
+        when :http then HTTP
         else fail ArgumentError, 'Invalid device type'
       end
     end

data/lib/logstash-logger/formatter/base.rb

@@ -4,23 +4,60 @@ require 'time'
 
 module LogStashLogger
   module Formatter
-    HOST = ::Socket.gethostname
+    HOST = {
+      'hostname' => ::Socket.gethostname,
+      'ip' => Socket.ip_address_list.reject(&:ipv4_loopback?).reject(&:ipv6_loopback?).map(&:ip_address)
+  }.freeze
 
     class Base < ::Logger::Formatter
+      FAILED_TO_FORMAT_MSG = 'Failed to format log event'
+      FAILED_TO_FORMAT_TAG = '_logstashlogger_format_failure'
+      attr_accessor :error_logger
       include ::LogStashLogger::TaggedLogging::Formatter
 
-      def initialize(customize_event: nil)
+      def initialize(customize_event: nil, error_logger: LogStashLogger.configuration.default_error_logger)
         @customize_event = customize_event
+        @error_logger = error_logger
         super()
       end
 
-      def call(severity, time, _progname, message)
+      def call(severity, time, progname, message)
         event = build_event(message, severity, time)
-        format_event(event) unless event.cancelled?
+        frame_event(event, progname) unless event.cancelled?
+      rescue StandardError => e
+        log_error(e)
+        fallback_event(severity, time, progname, e)
       end
 
       private
 
+      # Wraps a built event in this formatter's record framing. Formatters that
+      # add a header or trailer override this; the fallback below goes through
+      # the same framing.
+      def frame_event(event, _progname)
+        format_event(event)
+      end
+
+      # When a record cannot be built or serialized, emit a well-formed
+      # placeholder in this formatter's own framing instead, so the failure never
+      # changes how a collector splits the stream into records. Returns nil
+      # (which devices drop) if even the placeholder cannot be formatted.
+      def fallback_event(severity, time, progname, error)
+        event = LogStash::Event.new(
+          '@timestamp'.freeze => time.is_a?(Time) ? time : Time.now,
+          'message'.freeze => FAILED_TO_FORMAT_MSG,
+          'severity'.freeze => severity.to_s,
+          'host'.freeze => HOST,
+          'tags'.freeze => [FAILED_TO_FORMAT_TAG],
+          'error'.freeze => error.class.name
+        )
+        event.timestamp = event.timestamp.iso8601(3) if event.timestamp.is_a?(Time)
+        frame_event(event, progname)
+      rescue StandardError => e
+        log_error(e)
+        nil
+      end
+
       def build_event(message, severity, time)
         data = message
         if data.is_a?(String) && data.start_with?('{'.freeze)
@@ -31,15 +68,27 @@ module LogStashLogger
                   when LogStash::Event
                     data.clone
                   when Hash
-                    event_data = data.clone
-                    event_data['message'.freeze] = event_data.delete(:message) if event_data.key?(:message)
-                    event_data['tags'.freeze] = event_data.delete(:tags) if event_data.key?(:tags)
-                    event_data['source'.freeze] = event_data.delete(:source) if event_data.key?(:source)
-                    event_data['type'.freeze] = event_data.delete(:type) if event_data.key?(:type)
-                    event_data['@timestamp'.freeze] = time
+                    event_data = { '@timestamp'.freeze => time }
+                    data.each do |key, value|
+                      case key
+                      when :message, 'message'
+                        event_data['message'.freeze] = value
+                      when :tags, 'tags'
+                        event_data['tags'.freeze] = value
+                      when :source, 'source'
+                        event_data['source'.freeze] = value
+                      when :type, 'type'
+                        event_data['type'.freeze] = value
+                      when :'@timestamp', '@timestamp'
+                        # Keep the logger time unless the value is a usable time
+                        event_data['@timestamp'.freeze] = usable_time(value) || time
+                      else
+                        event_data[key] = value
+                      end
+                    end
                     LogStash::Event.new(event_data)
                   else
-                    LogStash::Event.new("message".freeze => msg2str(data), "@timestamp".freeze => time)
+                    LogStash::Event.new("@timestamp".freeze => time, "message".freeze => msg2str(data))
                 end
 
         event['severity'.freeze] ||= severity
@@ -68,6 +117,43 @@ module LogStashLogger
       def format_event(event)
         event
       end
+
+      # LogStash::Event runs String timestamps through Time.parse, which raises
+      # for text that is not a time. Keep every other value exactly as given.
+      def usable_time(value)
+        return value unless value.is_a?(String)
+        Time.parse(value)
+        value
+      rescue ArgumentError, RangeError
+        nil
+      end
+
+      def force_utf8_encoding(event)
+        original_message = event.instance_variable_get(:@data)['message']
+        event.message = original_message.dup.force_encoding(Encoding::UTF_8).scrub
+        event
+      end
+
+      # Check if the message has encoding issues that would cause JSON serialization problems.
+      # This is needed because some Ruby implementations (e.g., JRuby) may not raise
+      # exceptions during JSON encoding but produce malformed output instead.
+      def message_has_encoding_issue?(event)
+        message = event.instance_variable_get(:@data)['message']
+        return false unless message.is_a?(String)
+
+        # Check if already valid UTF-8
+        return false if message.encoding == Encoding::UTF_8 && message.valid_encoding?
+
+        # Try to encode to UTF-8 to detect issues
+        message.encode(Encoding::UTF_8)
+        false
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        true
+      end
+
+      def log_error(e)
+        error_logger.error "[#{self.class}] #{e.class} - #{e.message}"
+      end
     end
   end
 end