logstash-logger-p 0.26.1

data/lib/logstash-logger/device/connectable.rb

@@ -0,0 +1,131 @@
+require 'logstash-logger/buffer'
+
+module LogStashLogger
+  module Device
+    class Connectable < Base
+      include LogStashLogger::Buffer
+
+      attr_accessor :buffer_logger
+
+      def initialize(opts = {})
+        super
+
+        if opts[:batch_events]
+          warn "The :batch_events option is deprecated. Please use :buffer_max_items instead"
+        end
+
+        if opts[:batch_timeout]
+          warn "The :batch_timeout option is deprecated. Please use :buffer_max_interval instead"
+        end
+
+        @buffer_group = nil
+        @buffer_max_items = opts[:batch_events] || opts[:buffer_max_items]
+        @buffer_max_interval = opts[:batch_timeout] || opts[:buffer_max_interval]
+        @drop_messages_on_flush_error =
+          if opts.key?(:drop_messages_on_flush_error)
+            opts.delete(:drop_messages_on_flush_error)
+          else
+            false
+          end
+
+        @drop_messages_on_full_buffer =
+          if opts.key?(:drop_messages_on_full_buffer)
+            opts.delete(:drop_messages_on_full_buffer)
+          else
+            true
+          end
+
+        @buffer_flush_at_exit =
+          if opts.key?(:buffer_flush_at_exit)
+            opts.delete(:buffer_flush_at_exit)
+          else
+            true
+          end
+
+        @buffer_logger = opts[:buffer_logger]
+
+        buffer_initialize(
+          max_items: @buffer_max_items,
+          max_interval: @buffer_max_interval,
+          logger: buffer_logger,
+          autoflush: @sync,
+          drop_messages_on_flush_error: @drop_messages_on_flush_error,
+          drop_messages_on_full_buffer: @drop_messages_on_full_buffer,
+          flush_at_exit: @buffer_flush_at_exit
+        )
+      end
+
+      def write(message)
+        buffer_receive(message, @buffer_group) unless message.nil?
+      end
+
+      def flush(*args)
+        if args.empty?
+          buffer_flush
+        else
+          messages, group = *args
+          write_batch(messages, group)
+        end
+      end
+
+      def on_full_buffer_receive(data)
+        log_warning("Buffer Full - #{data}")
+      end
+
+      def close(opts = {})
+        if opts.fetch(:flush, true)
+          buffer_flush(final: true)
+        end
+
+        super
+      end
+
+      def to_io
+        with_connection do
+          super
+        end
+      end
+
+      def connected?
+        !!@io
+      end
+
+      def write_one(message)
+        with_connection do
+          super
+        end
+      end
+
+      def write_batch(messages, group = nil)
+        with_connection do
+          super
+        end
+      end
+
+      # Implemented by subclasses
+      def connect
+        fail NotImplementedError
+      end
+
+      def reset
+        reset_buffer
+        close(flush: false)
+      end
+
+      def reconnect
+        close(flush: false)
+        connect
+      end
+
+      # Ensure the block is executed with a valid connection
+      def with_connection(&block)
+        connect unless connected?
+        yield
+      rescue => e
+        log_error(e)
+        close(flush: false)
+        raise
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/file.rb

@@ -0,0 +1,23 @@
+require 'fileutils'
+
+module LogStashLogger
+  module Device
+    class File < Base
+      def initialize(opts)
+        super
+        @path = opts[:path] || fail(ArgumentError, "Path is required")
+        open
+      end
+
+      def open
+        unless ::File.exist? ::File.dirname @path
+          ::FileUtils.mkdir_p ::File.dirname @path
+        end
+
+        @io = ::File.open @path, ::File::WRONLY | ::File::APPEND | ::File::CREAT
+        @io.binmode
+        @io.sync = self.sync
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/firehose.rb

@@ -0,0 +1,42 @@
+begin
+  require 'aws-sdk-firehose'
+rescue LoadError
+  require 'aws-sdk'
+end
+
+require 'logstash-logger/device/aws_stream'
+
+module LogStashLogger
+  module Device
+    class Firehose < AwsStream
+      @stream_class = ::Aws::Firehose::Client
+      @recoverable_error_codes = [
+        "ServiceUnavailable",
+        "InternalFailure",
+        "ServiceUnavailableException"
+      ].freeze
+
+      def transform_message(message)
+        {
+          data: message
+        }
+      end
+
+      def put_records(records)
+        @io.put_record_batch({
+          records: records,
+          delivery_stream_name: @stream
+        })
+      end
+
+      def is_successful_response(resp)
+        resp.failed_put_count == 0
+      end
+
+      def get_response_records(resp)
+        resp.request_responses
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/io.rb

@@ -0,0 +1,11 @@
+module LogStashLogger
+  module Device
+    class IO < Base
+      def initialize(opts)
+        super
+        @io = opts[:io] || fail(ArgumentError, 'IO is required')
+        @io.sync = sync unless sync.nil?
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/kafka.rb

@@ -0,0 +1,57 @@
+require 'poseidon'
+
+module LogStashLogger
+  module Device
+    class Kafka < Connectable
+
+      DEFAULT_HOST = 'localhost'
+      DEFAULT_PORT = 9092
+      DEFAULT_TOPIC = 'logstash'
+      DEFAULT_PRODUCER = 'logstash-logger'
+      DEFAULT_BACKOFF = 1
+
+      attr_accessor :hosts, :topic, :producer, :backoff
+
+      def initialize(opts)
+        super
+        host = opts[:host] || DEFAULT_HOST
+        port = opts[:port] || DEFAULT_PORT
+        @hosts = opts[:hosts] || host.split(',').map { |h| "#{h}:#{port}" }
+        @topic = opts[:path] || DEFAULT_TOPIC
+        @producer = opts[:producer] || DEFAULT_PRODUCER
+        @backoff = opts[:backoff] || DEFAULT_BACKOFF
+        @buffer_group = @topic
+      end
+
+      def connect
+        @io = ::Poseidon::Producer.new(@hosts, @producer)
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue ::Poseidon::Errors::ChecksumError, Poseidon::Errors::UnableToFetchMetadata => e
+        log_error(e)
+        log_warning("reconnect/retry")
+        sleep backoff if backoff
+        reconnect
+        retry
+      rescue => e
+        log_error(e)
+        log_warning("giving up")
+        close(flush: false)
+      end
+
+      def write_batch(messages, topic = nil)
+        topic ||= @topic
+        with_connection do
+          @io.send_messages messages.map { |message| Poseidon::MessageToSend.new(topic, message) }
+        end
+      end
+
+      def write_one(message, topic = nil)
+        write_batch([message], topic)
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/kinesis.rb

@@ -0,0 +1,44 @@
+begin
+  require 'aws-sdk-kinesis'
+rescue LoadError
+  require 'aws-sdk'
+end
+
+require 'logstash-logger/device/aws_stream'
+
+module LogStashLogger
+  module Device
+    class Kinesis < AwsStream
+      @stream_class = ::Aws::Kinesis::Client
+      @recoverable_error_codes = [
+        "ServiceUnavailable",
+        "Throttling",
+        "RequestExpired",
+        "ProvisionedThroughputExceededException"
+      ].freeze
+
+      def transform_message(message)
+        {
+          data: message,
+          partition_key: SecureRandom.uuid
+        }
+      end
+
+      def put_records(records)
+        @io.put_records({
+          records: records,
+          stream_name: @stream
+        })
+      end
+
+      def is_successful_response(resp)
+        resp.failed_record_count == 0
+      end
+
+      def get_response_records(resp)
+        resp.records
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/multi_delegator.rb

@@ -0,0 +1,36 @@
+# Container to allow writes to multiple devices
+
+# Code originally from:
+# http://stackoverflow.com/a/6410202
+
+module LogStashLogger
+  module Device
+    class MultiDelegator < Base
+      attr_reader :devices
+
+      def initialize(opts)
+        @io = self
+        @devices = create_devices(opts)
+        self.class.delegate(:write, :close, :close!, :flush)
+      end
+
+      private
+
+      def create_devices(opts)
+        output_configurations = opts.delete(:outputs)
+        output_configurations.map do |device_opts|
+          device_opts = opts.merge(device_opts)
+          Device.new(device_opts)
+        end
+      end
+
+      def self.delegate(*methods)
+        methods.each do |m|
+          define_method(m) do |*args|
+            @devices.each { |device| device.send(m, *args) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/redis.rb

@@ -0,0 +1,76 @@
+require 'redis'
+
+module LogStashLogger
+  module Device
+    class Redis < Connectable
+      DEFAULT_LIST = 'logstash'
+
+      attr_accessor :list
+
+      def initialize(opts)
+        super
+        @list = opts.delete(:list) || DEFAULT_LIST
+        @buffer_group = @list
+
+        normalize_path(opts)
+        delete_unknown_keywords(opts)
+
+        @redis_options = opts
+      end
+
+      def connect
+        @io = ::Redis.new(@redis_options)
+      end
+
+      def reconnect
+        @io.client.reconnect
+      rescue => e
+        log_error(e)
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue ::Redis::InheritedError
+        reconnect
+        retry
+      rescue => e
+        log_error(e)
+        close(flush: false)
+        raise
+      end
+
+      def write_batch(messages, list = nil)
+        list ||= @list
+        with_connection do
+          @io.rpush(list, messages)
+        end
+      end
+
+      def write_one(message, list = nil)
+        write_batch(message, list)
+      end
+
+      def close!
+        @io && @io.quit
+      end
+
+      private
+
+      def normalize_path(opts)
+        path = opts.fetch(:path, nil)
+        if path
+          opts[:db] = path.gsub("/", "").to_i unless path.empty?
+          opts.delete(:path)
+        end
+      end
+
+      def delete_unknown_keywords(opts)
+        # due to restrictions in redis client version >= 5
+        # Continous error is being logged for unknown keywords, at present there is only one i.e. :sync
+        # to prevent that adding following line :)
+        opts.delete(:sync)
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/socket.rb

@@ -0,0 +1,21 @@
+require 'socket'
+
+module LogStashLogger
+  module Device
+    class Socket < Connectable
+      DEFAULT_HOST = '0.0.0.0'
+
+      attr_reader :host, :port
+
+      def initialize(opts)
+        super
+        @port = opts[:port] || fail(ArgumentError, "Port is required")
+        @host = opts[:host] || DEFAULT_HOST
+      end
+
+      def unrecoverable_error?(e)
+        e.is_a?(Errno::EMSGSIZE) || super
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/stderr.rb

@@ -0,0 +1,13 @@
+module LogStashLogger
+  module Device
+    class Stderr < IO
+      def initialize(opts={})
+        super({io: $stderr}.merge(opts))
+      end
+
+      def close!
+        # no-op
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/stdout.rb

@@ -0,0 +1,14 @@
+module LogStashLogger
+  module Device
+    class Stdout < IO
+      def initialize(opts={})
+        super({io: $stdout}.merge(opts))
+      end
+
+      def close!
+        # no-op
+        # Calling $stdout.close would be a bad idea
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/tcp.rb

@@ -0,0 +1,86 @@
+require 'openssl'
+
+module LogStashLogger
+  module Device
+    class TCP < Socket
+      attr_reader :ssl_certificate
+
+      def initialize(opts)
+        super
+
+        @ssl_certificate = opts[:ssl_certificate]
+        @ssl_context = opts[:ssl_context]
+        @use_ssl = !!(@ssl_certificate || opts[:ssl_context])
+        @use_ssl = opts[:ssl_enable] if opts.has_key? :ssl_enable
+        if opts.has_key?(:use_ssl)
+          @use_ssl = opts[:use_ssl]
+          warn "[LogStashLogger] The use_ssl option is deprecated. Use ssl_enable instead."
+        end
+        @verify_hostname = opts.fetch(:verify_hostname, true)
+      end
+
+      def ssl_context
+        return unless use_ssl?
+        @ssl_context || certificate_context
+      end
+
+      def use_ssl?
+        @use_ssl
+      end
+
+      def connect
+        if use_ssl?
+          io.connect
+          verify_hostname!
+        end
+        io
+      end
+
+      def io
+        @io ||= if use_ssl?
+          ssl_io
+        else
+          tcp_io
+        end
+      end
+
+      protected
+
+      def tcp_io
+        TCPSocket.new(@host, @port).tap do |socket|
+          socket.sync = sync unless sync.nil?
+        end
+      end
+
+      def ssl_io
+        ssl_context ?
+          OpenSSL::SSL::SSLSocket.new(tcp_io, ssl_context) :
+          OpenSSL::SSL::SSLSocket.new(tcp_io)
+      end
+
+      def certificate_context
+        return unless @ssl_certificate
+        @certificate_context ||= OpenSSL::SSL::SSLContext.new.tap do |ctx|
+          ctx.set_params(cert: @ssl_certificate)
+        end
+      end
+
+      def verify_hostname?
+        return false unless ssl_context
+        !! @verify_hostname
+      end
+
+      def verify_hostname!
+        @io.post_connection_check(hostname) if verify_hostname?
+      end
+
+      def hostname
+        if String === @verify_hostname
+          @verify_hostname
+        else
+          @host
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/udp.rb

@@ -0,0 +1,12 @@
+module LogStashLogger
+  module Device
+    class UDP < Socket
+      def connect
+        @io = UDPSocket.new.tap do |socket|
+          socket.connect(@host, @port)
+          socket.sync = sync unless sync.nil?
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/unix.rb

@@ -0,0 +1,18 @@
+require 'socket'
+
+module LogStashLogger
+  module Device
+    class Unix < Connectable
+      def initialize(opts={})
+        super
+        @path = opts[:path] || fail(ArgumentError, "Path is required")
+      end
+
+      def connect
+        @io = ::UNIXSocket.new(@path).tap do |socket|
+          socket.sync = sync unless sync.nil?
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device.rb

@@ -0,0 +1,67 @@
+require 'logstash-logger/device/base'
+
+module LogStashLogger
+  module Device
+    DEFAULT_TYPE = :udp
+
+    autoload :Base, 'logstash-logger/device/base'
+    autoload :Connectable, 'logstash-logger/device/connectable'
+    autoload :Socket, 'logstash-logger/device/socket'
+    autoload :UDP, 'logstash-logger/device/udp'
+    autoload :TCP, 'logstash-logger/device/tcp'
+    autoload :Unix, 'logstash-logger/device/unix'
+    autoload :Redis, 'logstash-logger/device/redis'
+    autoload :Kafka, 'logstash-logger/device/kafka'
+    autoload :Kinesis, 'logstash-logger/device/kinesis'
+    autoload :Firehose, 'logstash-logger/device/firehose'
+    autoload :File, 'logstash-logger/device/file'
+    autoload :IO, 'logstash-logger/device/io'
+    autoload :Stdout, 'logstash-logger/device/stdout'
+    autoload :Stderr, 'logstash-logger/device/stderr'
+    autoload :Balancer, 'logstash-logger/device/balancer'
+    autoload :MultiDelegator, 'logstash-logger/device/multi_delegator'
+
+    def self.new(opts)
+      opts = opts.dup
+      build_device(opts)
+    end
+
+    def self.build_device(opts)
+      if parsed_uri_opts = parse_uri_config(opts)
+        opts.delete(:uri)
+        opts.merge!(parsed_uri_opts)
+      end
+
+      type = opts.delete(:type) || DEFAULT_TYPE
+
+      device_klass_for(type).new(opts)
+    end
+
+    def self.parse_uri_config(opts)
+      if uri = opts[:uri]
+        require 'uri'
+        parsed = ::URI.parse(uri)
+        {type: parsed.scheme, host: parsed.host, port: parsed.port, path: parsed.path}
+      end
+    end
+
+    def self.device_klass_for(type)
+      case type.to_sym
+        when :udp then UDP
+        when :tcp then TCP
+        when :unix then Unix
+        when :file then File
+        when :redis then Redis
+        when :kafka then Kafka
+        when :kinesis then Kinesis
+        when :firehose then Firehose
+        when :io then IO
+        when :stdout then Stdout
+        when :stderr then Stderr
+        when :multi_delegator then MultiDelegator
+        when :balancer then Balancer
+        else fail ArgumentError, 'Invalid device type'
+      end
+    end
+  end
+end