logstash-logger 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7b8e0765b2b048a5ef9dcb2c42d4e31f3805029
-  data.tar.gz: c6cbcdefe330fe9aa8da5c93c78bab7bb2ee7751
+  metadata.gz: 6a53c4f86f44f72b31d055db56d35e6e374cfe30
+  data.tar.gz: de52e5fcefe9adf476a5b104d92ee2b6d9f1b1e4
 SHA512:
-  metadata.gz: e137e5bae19157b9f716faa987766fb19dfc96791e11a5813ab6b3274599ae21a957a0069fc411d1fe84ef32a0dac82a96b696657ffdb1d587ca31f8199a1f1a
-  data.tar.gz: 8bb2ce9e6305d789087923d6c3bbf382aedb4e2562a139786d157bbe14c40ab492c58b23e2c7a9ad5bfeefaef911babf0b88c969f0702a90ed7763198139dcbc
+  metadata.gz: 861e72a87a6230f386f15b9b14d5df7f303a294f843cc841352693a57fbdc17f0e55e5b8f04ba19ed6a13a72d1737ede43323b3786226cb4621487a70a68b886
+  data.tar.gz: 2f1e95420c8e2a5bb69a2a36ab4f8e4f40c4500d834fb16be66c113f5b3645e188434ab04fbded1b805cd110a52754e82b71e6927269abf1a4afbfeac038d451

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.24.0
+
+- Adds support for AWS Firehose. [#121](https://github.com/dwbutler/logstash-logger/pull/121)
+
 ## 0.23.0
 
 - Adds support for SSL host verification to the TCP device. [#114](https://github.com/dwbutler/logstash-logger/pull/114)

data/README.md

@@ -9,7 +9,7 @@ writing to a file or syslog since Logstash can receive the structured data direc
 ## Features
 
 * Can write directly to a logstash listener over a UDP or TCP/SSL connection.
-* Can write to a file, Redis, Kafka, Kinesis, a unix socket, syslog, stdout, or stderr.
+* Can write to a file, Redis, Kafka, Kinesis, Firehose, a unix socket, syslog, stdout, or stderr.
 * Logger can take a string message, a hash, a `LogStash::Event`, an object, or a JSON string as input.
 * Events are automatically populated with message, timestamp, host, and severity.
 * Writes in logstash JSON format, but supports other formats as well.
@@ -572,6 +572,30 @@ config.logstash.aws_secret_access_key = 'ASKASKHLD1234123412341234'
 
 ```
 
+#### Firehose
+
+Add the aws-sdk gem to your Gemfile:
+
+    gem 'aws-sdk'
+
+```ruby
+# Required
+config.logstash.type = :firehose
+
+# Optional, will default to the 'logstash' delivery stream
+config.logstash.stream = 'my-stream-name'
+
+# Optional, will default to 'us-east-1'
+config.logstash.aws_region = 'us-west-2'
+
+# Optional, will default to the AWS_ACCESS_KEY_ID environment variable
+config.logstash.aws_access_key_id = 'ASKASKHLD12341'
+
+# Optional, will default to the AWS_SECRET_ACCESS_KEY environment variable
+config.logstash.aws_secret_access_key = 'ASKASKHLD1234123412341234'
+
+```
+
 #### File
 
 ```ruby

data/lib/logstash-logger/device.rb

@@ -13,6 +13,7 @@ module LogStashLogger
     autoload :Redis, 'logstash-logger/device/redis'
     autoload :Kafka, 'logstash-logger/device/kafka'
     autoload :Kinesis, 'logstash-logger/device/kinesis'
+    autoload :Firehose, 'logstash-logger/device/firehose'
     autoload :File, 'logstash-logger/device/file'
     autoload :IO, 'logstash-logger/device/io'
     autoload :Stdout, 'logstash-logger/device/stdout'
@@ -53,6 +54,7 @@ module LogStashLogger
         when :redis then Redis
         when :kafka then Kafka
         when :kinesis then Kinesis
+        when :firehose then Firehose
         when :io then IO
         when :stdout then Stdout
         when :stderr then Stderr

data/lib/logstash-logger/device/aws_stream.rb

@@ -0,0 +1,91 @@
+require 'aws-sdk'
+
+module LogStashLogger
+  module Device
+    class AwsStream < Connectable
+
+      DEFAULT_REGION = 'us-east-1'
+      DEFAULT_STREAM = 'logstash'
+
+      @stream_class = nil
+      @recoverable_error_codes = []
+
+      class << self
+        attr_accessor :stream_class, :recoverable_error_codes
+      end
+
+      attr_accessor :aws_region, :stream
+
+      def initialize(opts)
+        super
+        @access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
+        @secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
+        @aws_region = opts[:aws_region] || DEFAULT_REGION
+        @stream = opts[:stream] || DEFAULT_STREAM
+      end
+
+      def transform_message(message)
+        fail NotImplementedError
+      end
+
+      def put_records(records)
+        fail NotImplementedError
+      end
+
+      def is_successful_response(resp)
+        fail NotImplementedError
+      end
+
+      def get_response_records(resp)
+        fail NotImplementedError
+      end
+
+      def connect
+        @io = self.class.stream_class.new(
+          region: @aws_region,
+          credentials: ::Aws::Credentials.new(@access_key_id, @secret_access_key)
+        )
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue => e
+        log_error(e)
+        log_warning("giving up")
+        close(flush: false)
+      end
+
+      def write_batch(messages, group = nil)
+        records = messages.map{ |m| transform_message(m) }
+
+        with_connection do
+          resp = put_records(records)
+
+          # Put any failed records back into the buffer
+          if !is_successful_response(resp)
+            get_response_records(resp).each_with_index do |record, index|
+              if self.class.recoverable_error_codes.include?(record.error_code)
+                log_warning("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
+                log_warning("Retrying")
+                write(records[index][:data])
+              elsif !record.error_code.nil? && record.error_code != ''
+                log_error("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
+              end
+            end
+          end
+        end
+      end
+
+      def write_one(message)
+        write_batch([message])
+      end
+
+      def close!
+        @io = nil
+      end
+
+    end
+  end
+end
+

data/lib/logstash-logger/device/firehose.rb

@@ -0,0 +1,37 @@
+require 'aws-sdk'
+require 'logstash-logger/device/aws_stream'
+
+module LogStashLogger
+  module Device
+    class Firehose < AwsStream
+      @stream_class = ::Aws::Firehose::Client
+      @recoverable_error_codes = [
+        "ServiceUnavailable",
+        "InternalFailure",
+        "ServiceUnavailableException"
+      ].freeze
+
+      def transform_message(message)
+        {
+          data: message
+        }
+      end
+
+      def put_records(records)
+        @io.put_record_batch({
+          records: records,
+          delivery_stream_name: @stream
+        })
+      end
+
+      def is_successful_response(resp)
+        resp.failed_put_count == 0
+      end
+
+      def get_response_records(resp)
+        resp.request_responses
+      end
+
+    end
+  end
+end

data/lib/logstash-logger/device/kinesis.rb

@@ -1,80 +1,39 @@
 require 'aws-sdk'
+require 'logstash-logger/device/aws_stream'
 
 module LogStashLogger
   module Device
-    class Kinesis < Connectable
-
-      DEFAULT_REGION = 'us-east-1'
-      DEFAULT_STREAM = 'logstash'
-      RECOVERABLE_ERROR_CODES = [
+    class Kinesis < AwsStream
+      @stream_class = ::Aws::Kinesis::Client
+      @recoverable_error_codes = [
         "ServiceUnavailable",
         "Throttling",
         "RequestExpired",
         "ProvisionedThroughputExceededException"
-      ]
-
-      attr_accessor :aws_region, :stream
+      ].freeze
 
-      def initialize(opts)
-        super
-        @access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
-        @secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
-        @aws_region = opts[:aws_region] || DEFAULT_REGION
-        @stream = opts[:stream] || DEFAULT_STREAM
+      def transform_message(message)
+        {
+          data: message,
+          partition_key: SecureRandom.uuid
+        }
       end
 
-      def connect
-        @io = ::Aws::Kinesis::Client.new(
-          region: @aws_region,
-          credentials: ::Aws::Credentials.new(@access_key_id, @secret_access_key)
-        )
+      def put_records(records)
+        @io.put_records({
+          records: records,
+          stream_name: @stream
+        })
       end
 
-      def with_connection
-        connect unless connected?
-        yield
-      rescue => e
-        log_error(e)
-        log_warning("giving up")
-        close(flush: false)
+      def is_successful_response(resp)
+        resp.failed_record_count == 0
       end
 
-      def write_batch(messages, group = nil)
-        kinesis_records = messages.map do |message|
-          {
-            data: message,
-            partition_key: SecureRandom.uuid
-          }
-        end
-
-        with_connection do
-          resp = @io.put_records({
-            records: kinesis_records,
-            stream_name: @stream
-          })
-
-          # Put any failed records back into the buffer
-          if resp.failed_record_count != 0
-            resp.records.each_with_index do |record, index|
-              if RECOVERABLE_ERROR_CODES.include?(record.error_code)
-                log_warning("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
-                log_warning("Retrying")
-                write(kinesis_records[index][:data])
-              elsif !record.error_code.nil? && record.error_code != ''
-                log_error("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
-              end
-            end
-          end
-        end
+      def get_response_records(resp)
+        resp.records
       end
 
-      def write_one(message)
-        write_batch([message])
-      end
-
-      def close!
-        @io = nil
-      end
     end
   end
 end

data/lib/logstash-logger/version.rb

@@ -1,3 +1,3 @@
 module LogStashLogger
-  VERSION = "0.23.0"
+  VERSION = "0.24.0"
 end

data/spec/device/firehose_spec.rb

@@ -0,0 +1,45 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Kinesis do
+  include_context 'device'
+
+  let(:client) { double("Aws::Firehose::Client") }
+
+  before(:each) do
+    allow(Aws::Firehose::Client).to receive(:new) { client }
+  end
+
+  it "writes to a Firehose stream" do
+    response = ::Aws::Firehose::Types::PutRecordBatchOutput.new
+    response.failed_put_count = 0
+    response.request_responses = []
+    expect(client).to receive(:put_record_batch) { response }
+    firehose_device.write "foo"
+
+    expect(firehose_device).to be_connected
+    firehose_device.close!
+    expect(firehose_device).not_to be_connected
+  end
+
+  it "it puts records with recoverable errors back in the buffer" do
+    failed_record = ::Aws::Firehose::Types::PutRecordBatchResponseEntry.new
+    failed_record.error_code = "InternalFailure"
+    failed_record.error_message = "InternalFailure"
+    response = ::Aws::Firehose::Types::PutRecordBatchOutput.new
+    response.failed_put_count = 1
+    response.request_responses = [failed_record]
+
+    expect(client).to receive(:put_record_batch) { response }
+    expect(firehose_device).to receive(:write).with("foo")
+
+    firehose_device.write_one "foo"
+  end
+
+  it "defaults the AWS region to us-east-1" do
+    expect(firehose_device.aws_region).to eq('us-east-1')
+  end
+
+  it "defaults the Firehose stream to logstash" do
+    expect(firehose_device.stream).to eq('logstash')
+  end
+end

data/spec/spec_helper.rb

@@ -62,6 +62,7 @@ RSpec.shared_context 'device' do
   let(:redis_device) { LogStashLogger::Device.new(type: :redis, sync: true) }
   let(:kafka_device) { LogStashLogger::Device.new(type: :kafka, sync: true) }
   let(:kinesis_device) { LogStashLogger::Device.new(type: :kinesis, sync: true) }
+  let(:firehose_device) { LogStashLogger::Device.new(type: :firehose, sync: true) }
 
   let(:outputs) { [{type: :stdout}, {type: :io, io: io}] }
   let(:multi_delegator_device) { LogStashLogger::Device.new(type: :multi_delegator, outputs: outputs) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-logger
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - David Butler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-14 00:00:00.000000000 Z
+date: 2017-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -191,10 +191,12 @@ files:
 - lib/logstash-logger/buffer.rb
 - lib/logstash-logger/configuration.rb
 - lib/logstash-logger/device.rb
+- lib/logstash-logger/device/aws_stream.rb
 - lib/logstash-logger/device/balancer.rb
 - lib/logstash-logger/device/base.rb
 - lib/logstash-logger/device/connectable.rb
 - lib/logstash-logger/device/file.rb
+- lib/logstash-logger/device/firehose.rb
 - lib/logstash-logger/device/io.rb
 - lib/logstash-logger/device/kafka.rb
 - lib/logstash-logger/device/kinesis.rb
@@ -234,6 +236,7 @@ files:
 - spec/device/balancer_spec.rb
 - spec/device/connectable_spec.rb
 - spec/device/file_spec.rb
+- spec/device/firehose_spec.rb
 - spec/device/io_spec.rb
 - spec/device/kafka_spec.rb
 - spec/device/kinesis_spec.rb
@@ -290,6 +293,7 @@ test_files:
 - spec/device/balancer_spec.rb
 - spec/device/connectable_spec.rb
 - spec/device/file_spec.rb
+- spec/device/firehose_spec.rb
 - spec/device/io_spec.rb
 - spec/device/kafka_spec.rb
 - spec/device/kinesis_spec.rb