logstash-logger-yajl 0.27.0

data/lib/logstash-logger/device/io.rb

@@ -0,0 +1,11 @@
+module LogStashLogger
+  module Device
+    class IO < Base
+      def initialize(opts)
+        super
+        @io = opts[:io] || fail(ArgumentError, 'IO is required')
+        @io.sync = sync unless sync.nil?
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/kafka.rb

@@ -0,0 +1,57 @@
+require 'poseidon'
+
+module LogStashLogger
+  module Device
+    class Kafka < Connectable
+
+      DEFAULT_HOST = 'localhost'
+      DEFAULT_PORT = 9092
+      DEFAULT_TOPIC = 'logstash'
+      DEFAULT_PRODUCER = 'logstash-logger'
+      DEFAULT_BACKOFF = 1
+
+      attr_accessor :hosts, :topic, :producer, :backoff
+
+      def initialize(opts)
+        super
+        host = opts[:host] || DEFAULT_HOST
+        port = opts[:port] || DEFAULT_PORT
+        @hosts = opts[:hosts] || host.split(',').map { |h| "#{h}:#{port}" }
+        @topic = opts[:path] || DEFAULT_TOPIC
+        @producer = opts[:producer] || DEFAULT_PRODUCER
+        @backoff = opts[:backoff] || DEFAULT_BACKOFF
+        @buffer_group = @topic
+      end
+
+      def connect
+        @io = ::Poseidon::Producer.new(@hosts, @producer)
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue ::Poseidon::Errors::ChecksumError, Poseidon::Errors::UnableToFetchMetadata => e
+        log_error(e)
+        log_warning("reconnect/retry")
+        sleep backoff if backoff
+        reconnect
+        retry
+      rescue => e
+        log_error(e)
+        log_warning("giving up")
+        close(flush: false)
+      end
+
+      def write_batch(messages, topic = nil)
+        topic ||= @topic
+        with_connection do
+          @io.send_messages messages.map { |message| Poseidon::MessageToSend.new(topic, message) }
+        end
+      end
+
+      def write_one(message, topic = nil)
+        write_batch([message], topic)
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/kinesis.rb

@@ -0,0 +1,44 @@
+begin
+  require 'aws-sdk-kinesis'
+rescue LoadError
+  require 'aws-sdk'
+end
+
+require 'logstash-logger/device/aws_stream'
+
+module LogStashLogger
+  module Device
+    class Kinesis < AwsStream
+      @stream_class = ::Aws::Kinesis::Client
+      @recoverable_error_codes = [
+        "ServiceUnavailable",
+        "Throttling",
+        "RequestExpired",
+        "ProvisionedThroughputExceededException"
+      ].freeze
+
+      def transform_message(message)
+        {
+          data: message,
+          partition_key: SecureRandom.uuid
+        }
+      end
+
+      def put_records(records)
+        @io.put_records({
+          records: records,
+          stream_name: @stream
+        })
+      end
+
+      def is_successful_response(resp)
+        resp.failed_record_count == 0
+      end
+
+      def get_response_records(resp)
+        resp.records
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/multi_delegator.rb

@@ -0,0 +1,36 @@
+# Container to allow writes to multiple devices
+
+# Code originally from:
+# http://stackoverflow.com/a/6410202
+
+module LogStashLogger
+  module Device
+    class MultiDelegator < Base
+      attr_reader :devices
+
+      def initialize(opts)
+        @io = self
+        @devices = create_devices(opts)
+        self.class.delegate(:write, :close, :close!, :flush)
+      end
+
+      private
+
+      def create_devices(opts)
+        output_configurations = opts.delete(:outputs)
+        output_configurations.map do |device_opts|
+          device_opts = opts.merge(device_opts)
+          Device.new(device_opts)
+        end
+      end
+
+      def self.delegate(*methods)
+        methods.each do |m|
+          define_method(m) do |*args|
+            @devices.each { |device| device.send(m, *args) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/redis.rb

@@ -0,0 +1,69 @@
+require 'redis'
+
+module LogStashLogger
+  module Device
+    class Redis < Connectable
+      DEFAULT_LIST = 'logstash'
+
+      attr_accessor :list
+
+      def initialize(opts)
+        super
+        @list = opts.delete(:list) || DEFAULT_LIST
+        @buffer_group = @list
+
+        normalize_path(opts)
+
+        @redis_options = opts
+      end
+
+      def connect
+        @io = ::Redis.new(@redis_options)
+      end
+
+      def reconnect
+        @io.client.reconnect
+      rescue => e
+        log_error(e)
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue ::Redis::InheritedError
+        reconnect
+        retry
+      rescue => e
+        log_error(e)
+        close(flush: false)
+        raise
+      end
+
+      def write_batch(messages, list = nil)
+        list ||= @list
+        with_connection do
+          @io.rpush(list, messages)
+        end
+      end
+
+      def write_one(message, list = nil)
+        write_batch(message, list)
+      end
+
+      def close!
+        @io && @io.quit
+      end
+
+      private
+
+      def normalize_path(opts)
+        path = opts.fetch(:path, nil)
+        if path
+          opts[:db] = path.gsub("/", "").to_i unless path.empty?
+          opts.delete(:path)
+        end
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/socket.rb

@@ -0,0 +1,21 @@
+require 'socket'
+
+module LogStashLogger
+  module Device
+    class Socket < Connectable
+      DEFAULT_HOST = '0.0.0.0'
+
+      attr_reader :host, :port
+
+      def initialize(opts)
+        super
+        @port = opts[:port] || fail(ArgumentError, "Port is required")
+        @host = opts[:host] || DEFAULT_HOST
+      end
+
+      def unrecoverable_error?(e)
+        e.is_a?(Errno::EMSGSIZE) || super
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/stderr.rb

@@ -0,0 +1,13 @@
+module LogStashLogger
+  module Device
+    class Stderr < IO
+      def initialize(opts={})
+        super({io: $stderr}.merge(opts))
+      end
+
+      def close!
+        # no-op
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/stdout.rb

@@ -0,0 +1,14 @@
+module LogStashLogger
+  module Device
+    class Stdout < IO
+      def initialize(opts={})
+        super({io: $stdout}.merge(opts))
+      end
+
+      def close!
+        # no-op
+        # Calling $stdout.close would be a bad idea
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/tcp.rb

@@ -0,0 +1,86 @@
+require 'openssl'
+
+module LogStashLogger
+  module Device
+    class TCP < Socket
+      attr_reader :ssl_certificate
+
+      def initialize(opts)
+        super
+
+        @ssl_certificate = opts[:ssl_certificate]
+        @ssl_context = opts[:ssl_context]
+        @use_ssl = !!(@ssl_certificate || opts[:ssl_context])
+        @use_ssl = opts[:ssl_enable] if opts.has_key? :ssl_enable
+        if opts.has_key?(:use_ssl)
+          @use_ssl = opts[:use_ssl]
+          warn "[LogStashLogger] The use_ssl option is deprecated. Use ssl_enable instead."
+        end
+        @verify_hostname = opts.fetch(:verify_hostname, true)
+      end
+
+      def ssl_context
+        return unless use_ssl?
+        @ssl_context || certificate_context
+      end
+
+      def use_ssl?
+        @use_ssl
+      end
+
+      def connect
+        if use_ssl?
+          io.connect
+          verify_hostname!
+        end
+        io
+      end
+
+      def io
+        @io ||= if use_ssl?
+          ssl_io
+        else
+          tcp_io
+        end
+      end
+
+      protected
+
+      def tcp_io
+        TCPSocket.new(@host, @port).tap do |socket|
+          socket.sync = sync unless sync.nil?
+        end
+      end
+
+      def ssl_io
+        ssl_context ?
+          OpenSSL::SSL::SSLSocket.new(tcp_io, ssl_context) :
+          OpenSSL::SSL::SSLSocket.new(tcp_io)
+      end
+
+      def certificate_context
+        return unless @ssl_certificate
+        @certificate_context ||= OpenSSL::SSL::SSLContext.new.tap do |ctx|
+          ctx.set_params(cert: @ssl_certificate)
+        end
+      end
+
+      def verify_hostname?
+        return false unless ssl_context
+        !! @verify_hostname
+      end
+
+      def verify_hostname!
+        @io.post_connection_check(hostname) if verify_hostname?
+      end
+
+      def hostname
+        if String === @verify_hostname
+          @verify_hostname
+        else
+          @host
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/udp.rb

@@ -0,0 +1,12 @@
+module LogStashLogger
+  module Device
+    class UDP < Socket
+      def connect
+        @io = UDPSocket.new.tap do |socket|
+          socket.connect(@host, @port)
+          socket.sync = sync unless sync.nil?
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/unix.rb

@@ -0,0 +1,18 @@
+require 'socket'
+
+module LogStashLogger
+  module Device
+    class Unix < Connectable
+      def initialize(opts={})
+        super
+        @path = opts[:path] || fail(ArgumentError, "Path is required")
+      end
+
+      def connect
+        @io = ::UNIXSocket.new(@path).tap do |socket|
+          socket.sync = sync unless sync.nil?
+        end
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter.rb

@@ -0,0 +1,51 @@
+require 'logstash-logger/formatter/base'
+
+module LogStashLogger
+	module Formatter
+		DEFAULT_FORMATTER = :json_lines
+
+    autoload :LogStashEvent, 'logstash-logger/formatter/logstash_event'
+    autoload :Json, 'logstash-logger/formatter/json'
+    autoload :JsonLines, 'logstash-logger/formatter/json_lines'
+    autoload :Cee, 'logstash-logger/formatter/cee'
+    autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
+
+    def self.new(formatter_type, customize_event: nil)
+      build_formatter(formatter_type, customize_event)
+    end
+
+    def self.build_formatter(formatter_type, customize_event)
+      formatter_type ||= DEFAULT_FORMATTER
+
+      formatter = if custom_formatter_instance?(formatter_type)
+        formatter_type
+      elsif custom_formatter_class?(formatter_type)
+        formatter_type.new
+      else
+        formatter_klass(formatter_type).new(customize_event: customize_event)
+      end
+
+      formatter.send(:extend, ::LogStashLogger::TaggedLogging::Formatter)
+      formatter
+    end
+
+    def self.formatter_klass(formatter_type)
+      case formatter_type.to_sym
+      when :json_lines then JsonLines
+      when :json then Json
+      when :logstash_event then LogStashEvent
+      when :cee then Cee
+      when :cee_syslog then CeeSyslog
+      else fail ArgumentError, 'Invalid formatter'
+      end
+    end
+
+    def self.custom_formatter_instance?(formatter_type)
+      formatter_type.respond_to?(:call)
+    end
+
+    def self.custom_formatter_class?(formatter_type)
+      formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
+    end
+	end
+end