logstash-logger-yajl 0.27.0

data/lib/logstash-logger/device.rb

@@ -0,0 +1,67 @@
+require 'logstash-logger/device/base'
+
+module LogStashLogger
+  module Device
+    DEFAULT_TYPE = :udp
+
+    autoload :Base, 'logstash-logger/device/base'
+    autoload :Connectable, 'logstash-logger/device/connectable'
+    autoload :Socket, 'logstash-logger/device/socket'
+    autoload :UDP, 'logstash-logger/device/udp'
+    autoload :TCP, 'logstash-logger/device/tcp'
+    autoload :Unix, 'logstash-logger/device/unix'
+    autoload :Redis, 'logstash-logger/device/redis'
+    autoload :Kafka, 'logstash-logger/device/kafka'
+    autoload :Kinesis, 'logstash-logger/device/kinesis'
+    autoload :Firehose, 'logstash-logger/device/firehose'
+    autoload :File, 'logstash-logger/device/file'
+    autoload :IO, 'logstash-logger/device/io'
+    autoload :Stdout, 'logstash-logger/device/stdout'
+    autoload :Stderr, 'logstash-logger/device/stderr'
+    autoload :Balancer, 'logstash-logger/device/balancer'
+    autoload :MultiDelegator, 'logstash-logger/device/multi_delegator'
+
+    def self.new(opts)
+      opts = opts.dup
+      build_device(opts)
+    end
+
+    def self.build_device(opts)
+      if parsed_uri_opts = parse_uri_config(opts)
+        opts.delete(:uri)
+        opts.merge!(parsed_uri_opts)
+      end
+
+      type = opts.delete(:type) || DEFAULT_TYPE
+
+      device_klass_for(type).new(opts)
+    end
+
+    def self.parse_uri_config(opts)
+      if uri = opts[:uri]
+        require 'uri'
+        parsed = ::URI.parse(uri)
+        {type: parsed.scheme, host: parsed.host, port: parsed.port, path: parsed.path}
+      end
+    end
+
+    def self.device_klass_for(type)
+      case type.to_sym
+        when :udp then UDP
+        when :tcp then TCP
+        when :unix then Unix
+        when :file then File
+        when :redis then Redis
+        when :kafka then Kafka
+        when :kinesis then Kinesis
+        when :firehose then Firehose
+        when :io then IO
+        when :stdout then Stdout
+        when :stderr then Stderr
+        when :multi_delegator then MultiDelegator
+        when :balancer then Balancer
+        else fail ArgumentError, 'Invalid device type'
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/aws_stream.rb

@@ -0,0 +1,94 @@
+begin
+  require 'aws-sdk-core'
+rescue LoadError
+  require 'aws-sdk'
+end
+
+module LogStashLogger
+  module Device
+    class AwsStream < Connectable
+
+      DEFAULT_STREAM = 'logstash'
+
+      @stream_class = nil
+      @recoverable_error_codes = []
+
+      class << self
+        attr_accessor :stream_class, :recoverable_error_codes
+      end
+
+      attr_accessor :aws_region, :stream
+
+      def initialize(opts)
+        super
+        @access_key_id = opts[:aws_access_key_id]
+        @secret_access_key = opts[:aws_secret_access_key]
+        @aws_region = opts[:aws_region]
+        @stream = opts[:stream] || DEFAULT_STREAM
+      end
+
+      def transform_message(message)
+        fail NotImplementedError
+      end
+
+      def put_records(records)
+        fail NotImplementedError
+      end
+
+      def is_successful_response(resp)
+        fail NotImplementedError
+      end
+
+      def get_response_records(resp)
+        fail NotImplementedError
+      end
+
+      def connect
+        client_opts = {}
+        client_opts[:credentials] = Aws::Credentials.new(@access_key_id, @secret_access_key) unless @access_key_id == nil || @secret_access_key == nil
+        client_opts[:region] = @aws_region unless @aws_region == nil
+        @io = self.class.stream_class.new(client_opts)
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue => e
+        log_error(e)
+        log_warning("giving up")
+        close(flush: false)
+      end
+
+      def write_batch(messages, group = nil)
+        records = messages.map{ |m| transform_message(m) }
+
+        with_connection do
+          resp = put_records(records)
+
+          # Put any failed records back into the buffer
+          if !is_successful_response(resp)
+            get_response_records(resp).each_with_index do |record, index|
+              if self.class.recoverable_error_codes.include?(record.error_code)
+                log_warning("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
+                log_warning("Retrying")
+                write(records[index][:data])
+              elsif !record.error_code.nil? && record.error_code != ''
+                log_error("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
+              end
+            end
+          end
+        end
+      end
+
+      def write_one(message)
+        write_batch([message])
+      end
+
+      def close!
+        @io = nil
+      end
+
+    end
+  end
+end
+

data/lib/logstash-logger/device/balancer.rb

@@ -0,0 +1,40 @@
+module LogStashLogger
+  module Device
+    class Balancer < Base
+      attr_reader :devices
+
+      def initialize(opts)
+        @io = self
+        @devices = create_devices(opts[:outputs])
+        self.class.delegate_to_all(:close, :close!, :flush)
+        self.class.delegate_to_one(:write)
+      end
+
+      private
+
+      def create_devices(opts)
+        opts.map { |device_opts| Device.new(device_opts) }
+      end
+
+      def self.delegate_to_all(*methods)
+        methods.each do |m|
+          define_method(m) do |*args|
+            devices.each { |device| device.send(m, *args) }
+          end
+        end
+      end
+
+      def self.delegate_to_one(*methods)
+        methods.each do |m|
+          define_method(m) do |*args|
+            select_device.send(m, *args)
+          end
+        end
+      end
+
+      def select_device
+        devices.sample
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/base.rb

@@ -0,0 +1,73 @@
+module LogStashLogger
+  module Device
+    class Base
+      attr_reader :io
+      attr_accessor :sync
+      attr_accessor :error_logger
+
+      def initialize(opts={})
+        @sync = opts[:sync]
+        @error_logger = opts.fetch(:error_logger, LogStashLogger.configuration.default_error_logger)
+      end
+
+      def to_io
+        @io
+      end
+
+      def write(message)
+        write_one(message) unless message.nil?
+      end
+
+      def write_one(message)
+        @io.write(message)
+      rescue => e
+        if unrecoverable_error?(e)
+          log_error(e)
+          log_warning("unrecoverable error, aborting write")
+        else
+          raise
+        end
+      end
+
+      def write_batch(messages, group = nil)
+        messages.each do |message|
+          write_one(message)
+        end
+      end
+
+      def flush
+        @io && @io.flush
+      end
+
+      def reset
+        close
+      end
+
+      def close(opts = {})
+        close!
+      rescue => e
+        log_error(e)
+      end
+
+      def close!
+        @io && @io.close
+      ensure
+        @io = nil
+      end
+
+      def unrecoverable_error?(e)
+        e.is_a?(JSON::GeneratorError)
+      end
+
+      private
+
+      def log_error(e)
+        error_logger.error "[#{self.class}] #{e.class} - #{e.message}"
+      end
+
+      def log_warning(message)
+        error_logger.warn("[#{self.class}] #{message}")
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/connectable.rb

@@ -0,0 +1,131 @@
+require 'logstash-logger/buffer'
+
+module LogStashLogger
+  module Device
+    class Connectable < Base
+      include LogStashLogger::Buffer
+
+      attr_accessor :buffer_logger
+
+      def initialize(opts = {})
+        super
+
+        if opts[:batch_events]
+          warn "The :batch_events option is deprecated. Please use :buffer_max_items instead"
+        end
+
+        if opts[:batch_timeout]
+          warn "The :batch_timeout option is deprecated. Please use :buffer_max_interval instead"
+        end
+
+        @buffer_group = nil
+        @buffer_max_items = opts[:batch_events] || opts[:buffer_max_items]
+        @buffer_max_interval = opts[:batch_timeout] || opts[:buffer_max_interval]
+        @drop_messages_on_flush_error =
+          if opts.key?(:drop_messages_on_flush_error)
+            opts.delete(:drop_messages_on_flush_error)
+          else
+            false
+          end
+
+        @drop_messages_on_full_buffer =
+          if opts.key?(:drop_messages_on_full_buffer)
+            opts.delete(:drop_messages_on_full_buffer)
+          else
+            true
+          end
+
+        @buffer_flush_at_exit =
+          if opts.key?(:buffer_flush_at_exit)
+            opts.delete(:buffer_flush_at_exit)
+          else
+            true
+          end
+
+        @buffer_logger = opts[:buffer_logger]
+
+        buffer_initialize(
+          max_items: @buffer_max_items,
+          max_interval: @buffer_max_interval,
+          logger: buffer_logger,
+          autoflush: @sync,
+          drop_messages_on_flush_error: @drop_messages_on_flush_error,
+          drop_messages_on_full_buffer: @drop_messages_on_full_buffer,
+          flush_at_exit: @buffer_flush_at_exit
+        )
+      end
+
+      def write(message)
+        buffer_receive(message, @buffer_group) unless message.nil?
+      end
+
+      def flush(*args)
+        if args.empty?
+          buffer_flush
+        else
+          messages, group = *args
+          write_batch(messages, group)
+        end
+      end
+
+      def on_full_buffer_receive(data)
+        log_warning("Buffer Full - #{data}")
+      end
+
+      def close(opts = {})
+        if opts.fetch(:flush, true)
+          buffer_flush(final: true)
+        end
+
+        super
+      end
+
+      def to_io
+        with_connection do
+          super
+        end
+      end
+
+      def connected?
+        !!@io
+      end
+
+      def write_one(message)
+        with_connection do
+          super
+        end
+      end
+
+      def write_batch(messages, group = nil)
+        with_connection do
+          super
+        end
+      end
+
+      # Implemented by subclasses
+      def connect
+        fail NotImplementedError
+      end
+
+      def reset
+        reset_buffer
+        close(flush: false)
+      end
+
+      def reconnect
+        close(flush: false)
+        connect
+      end
+
+      # Ensure the block is executed with a valid connection
+      def with_connection(&block)
+        connect unless connected?
+        yield
+      rescue => e
+        log_error(e)
+        close(flush: false)
+        raise
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/file.rb

@@ -0,0 +1,23 @@
+require 'fileutils'
+
+module LogStashLogger
+  module Device
+    class File < Base
+      def initialize(opts)
+        super
+        @path = opts[:path] || fail(ArgumentError, "Path is required")
+        open
+      end
+
+      def open
+        unless ::File.exist? ::File.dirname @path
+          ::FileUtils.mkdir_p ::File.dirname @path
+        end
+
+        @io = ::File.open @path, ::File::WRONLY | ::File::APPEND | ::File::CREAT
+        @io.binmode
+        @io.sync = self.sync
+      end
+    end
+  end
+end

data/lib/logstash-logger/device/firehose.rb

@@ -0,0 +1,42 @@
+begin
+  require 'aws-sdk-firehose'
+rescue LoadError
+  require 'aws-sdk'
+end
+
+require 'logstash-logger/device/aws_stream'
+
+module LogStashLogger
+  module Device
+    class Firehose < AwsStream
+      @stream_class = ::Aws::Firehose::Client
+      @recoverable_error_codes = [
+        "ServiceUnavailable",
+        "InternalFailure",
+        "ServiceUnavailableException"
+      ].freeze
+
+      def transform_message(message)
+        {
+          data: message
+        }
+      end
+
+      def put_records(records)
+        @io.put_record_batch({
+          records: records,
+          delivery_stream_name: @stream
+        })
+      end
+
+      def is_successful_response(resp)
+        resp.failed_put_count == 0
+      end
+
+      def get_response_records(resp)
+        resp.request_responses
+      end
+
+    end
+  end
+end