logstash-logger 0.21.0 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 514e39db2da9a557f7ae7d76f0983d1d0df0ad0e
-  data.tar.gz: 5f1b91eaf92734f575210e0b47605d18b9e9a009
+  metadata.gz: 1139c199fb7c041dad2f2e0469492a51f6812127
+  data.tar.gz: 5133b64e944b70de35eadab44180a6bcbb99b832
 SHA512:
-  metadata.gz: 1cd1f25d9c54d7819ffc8c5a8707de72df1f46ac61bf410c6f7777eb6364aca34378e7cce99701e69eeb8febb1a15e433af2fbac39bd74d43e9dcd068bbbcdb1
-  data.tar.gz: 19eff3e6e2b01ddf299060372dc6caeae1a9fe148f8bb04d2e8d3e10f93e16e80229a3ef1819dbb08a81eda59fd53b13cf2552ab1bd05078702b56b055cc3709
+  metadata.gz: b1cdc025812374e95d3fb0e2f4322a3e4c7c79db328e79f4a8bfe50f93f3d9e637703297db3e4f5d3f0c4a3d7c84ee346a9d792e79e9fef292d36332b41ee3af
+  data.tar.gz: 84c4c6a90f7430215cbb36f7e52d3af8a667aaa9bb16a1ac09f3cfa648efe99dfc329209a262d619642f0419ad52c991bbe1c963f4d7097aaf5972509162078f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.22.0
+
+- Adds support for writing to Amazon Kinesis.  [#111](https://github.com/dwbutler/logstash-logger/pull/111)
+
 ## 0.21.0
 
 - Support for merging top level configuration in MultiLogger and
@@ -8,6 +12,7 @@
 - Fixes missing require for URI [#107](https://github.com/dwbutler/logstash-logger/pull/107)
 
 ## 0.20.0
+
 - Adds support for logging debug/error information for the connectable device's buffer. [#102](https://github.com/dwbutler/logstash-logger/pull/102)
 
 ## 0.19.2

data/README.md

@@ -8,7 +8,7 @@ writing to a file or syslog since logstash can receive the structured data direc
 ## Features
 
 * Can write directly to a logstash listener over a UDP or TCP/SSL connection.
-* Can write to a file, Redis, Kafka, a unix socket, syslog, stdout, or stderr.
+* Can write to a file, Redis, Kafka, Kinesis, a unix socket, syslog, stdout, or stderr.
 * Logger can take a string message, a hash, a `LogStash::Event`, an object, or a JSON string as input.
 * Events are automatically populated with message, timestamp, host, and severity.
 * Writes in logstash JSON format, but supports other formats as well.
@@ -491,6 +491,30 @@ config.logstash.backoff = 1
 
 ```
 
+#### Kinesis
+
+Add the aws-sdk gem to your Gemfile:
+
+    gem 'aws-sdk'
+
+```ruby
+# Required
+config.logstash.type = :kinesis
+
+# Optional, will default to the 'logstash' stream
+config.logstash.stream = 'my-stream-name'
+
+# Optional, will default to 'us-east-1'
+config.logstash.aws_region = 'us-west-2'
+
+# Optional, will default to the AWS_ACCESS_KEY_ID environment variable
+config.logstash.aws_access_key_id = 'ASKASKHLD12341'
+
+# Optional, will default to the AWS_SECRET_ACCESS_KEY environment variable
+config.logstash.aws_secret_access_key = 'ASKASKHLD1234123412341234'
+
+```
+
 #### File
 
 ```ruby
@@ -694,6 +718,7 @@ logger = LogStashLogger.new('localhost', 5228, :tcp)
 * [Vitaly Gorodetsky](https://github.com/vitalis)
 * [Courtland Caldwell](https://github.com/caldwecr)
 * [Bibek Shrestha](https://github.com/bibstha)
+* [Alex Ianus](https://github.com/aianus)
 
 ## Contributing
 

data/lib/logstash-logger/device.rb

@@ -12,6 +12,7 @@ module LogStashLogger
     autoload :Unix, 'logstash-logger/device/unix'
     autoload :Redis, 'logstash-logger/device/redis'
     autoload :Kafka, 'logstash-logger/device/kafka'
+    autoload :Kinesis, 'logstash-logger/device/kinesis'
     autoload :File, 'logstash-logger/device/file'
     autoload :IO, 'logstash-logger/device/io'
     autoload :Stdout, 'logstash-logger/device/stdout'
@@ -51,6 +52,7 @@ module LogStashLogger
         when :file then File
         when :redis then Redis
         when :kafka then Kafka
+        when :kinesis then Kinesis
         when :io then IO
         when :stdout then Stdout
         when :stderr then Stderr

data/lib/logstash-logger/device/kinesis.rb

@@ -0,0 +1,80 @@
+require 'aws-sdk'
+
+module LogStashLogger
+  module Device
+    class Kinesis < Connectable
+
+      DEFAULT_REGION = 'us-east-1'
+      DEFAULT_STREAM = 'logstash'
+      RECOVERABLE_ERROR_CODES = [
+        "ServiceUnavailable",
+        "Throttling",
+        "RequestExpired",
+        "ProvisionedThroughputExceededException"
+      ]
+
+      attr_accessor :aws_region, :stream
+
+      def initialize(opts)
+        super
+        @access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
+        @secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
+        @aws_region = opts[:aws_region] || DEFAULT_REGION
+        @stream = opts[:stream] || DEFAULT_STREAM
+      end
+
+      def connect
+        @io = ::Aws::Kinesis::Client.new(
+          region: @aws_region,
+          credentials: ::Aws::Credentials.new(@access_key_id, @secret_access_key)
+        )
+      end
+
+      def with_connection
+        connect unless connected?
+        yield
+      rescue => e
+        log_error(e)
+        log_warning("giving up")
+        close(flush: false)
+      end
+
+      def write_batch(messages, group = nil)
+        kinesis_records = messages.map do |message|
+          {
+            data: message,
+            partition_key: SecureRandom.uuid
+          }
+        end
+
+        with_connection do
+          resp = @io.put_records({
+            records: kinesis_records,
+            stream_name: @stream
+          })
+
+          # Put any failed records back into the buffer
+          if resp.failed_record_count != 0
+            resp.records.each_with_index do |record, index|
+              if RECOVERABLE_ERROR_CODES.include?(record.error_code)
+                log_warning("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
+                log_warning("Retrying")
+                write(kinesis_records[index][:data])
+              elsif !record.error_code.nil? && record.error_code != ''
+                log_error("Failed to post record to kinesis with error: #{record.error_code} #{record.error_message}")
+              end
+            end
+          end
+        end
+      end
+
+      def write_one(message)
+        write_batch([message])
+      end
+
+      def close!
+        @io = nil
+      end
+    end
+  end
+end

data/lib/logstash-logger/version.rb

@@ -1,3 +1,3 @@
 module LogStashLogger
-  VERSION = "0.21.0"
+  VERSION = "0.22.0"
 end

data/logstash-logger.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |gem|
   end
   gem.add_development_dependency 'redis'
   gem.add_development_dependency 'poseidon'
+  gem.add_development_dependency 'aws-sdk'
 
   if RUBY_VERSION < '2' || defined?(JRUBY_VERSION)
     gem.add_development_dependency 'SyslogLogger'
@@ -35,6 +36,10 @@ Gem::Specification.new do |gem|
     gem.add_development_dependency 'json', '~> 1.8'
   end
 
+  if RUBY_VERSION < '2.1'
+    gem.add_development_dependency 'nokogiri', '~> 1.6.8'
+  end
+
   gem.add_development_dependency 'rspec', '>= 3'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'pry'

data/spec/device/kinesis_spec.rb

@@ -0,0 +1,45 @@
+require 'logstash-logger'
+
+describe LogStashLogger::Device::Kinesis do
+  include_context 'device'
+
+  let(:client) { double("Aws::Kinesis::Client") }
+
+  before(:each) do
+    allow(Aws::Kinesis::Client).to receive(:new) { client }
+  end
+
+  it "writes to a Kinesis stream" do
+    response = ::Aws::Kinesis::Types::PutRecordsOutput.new
+    response.failed_record_count = 0
+    response.records = []
+    expect(client).to receive(:put_records) { response }
+    kinesis_device.write "foo"
+
+    expect(kinesis_device).to be_connected
+    kinesis_device.close!
+    expect(kinesis_device).not_to be_connected
+  end
+
+  it "it puts records with recoverable errors back in the buffer" do
+    failed_record = ::Aws::Kinesis::Types::PutRecordsResultEntry.new
+    failed_record.error_code = "ProvisionedThroughputExceededException"
+    failed_record.error_message = "ProvisionedThroughputExceededException"
+    response = ::Aws::Kinesis::Types::PutRecordsOutput.new
+    response.failed_record_count = 1
+    response.records = [failed_record]
+
+    expect(client).to receive(:put_records) { response }
+    expect(kinesis_device).to receive(:write).with("foo")
+
+    kinesis_device.write_one "foo"
+  end
+
+  it "defaults the AWS region to us-east-1" do
+    expect(kinesis_device.aws_region).to eq('us-east-1')
+  end
+
+  it "defaults the kinesis stream to logstash" do
+    expect(kinesis_device.stream).to eq('logstash')
+  end
+end

data/spec/spec_helper.rb

@@ -61,6 +61,7 @@ RSpec.shared_context 'device' do
 
   let(:redis_device) { LogStashLogger::Device.new(type: :redis, sync: true) }
   let(:kafka_device) { LogStashLogger::Device.new(type: :kafka, sync: true) }
+  let(:kinesis_device) { LogStashLogger::Device.new(type: :kinesis, sync: true) }
 
   let(:outputs) { [{type: :stdout}, {type: :io, io: io}] }
   let(:multi_delegator_device) { LogStashLogger::Device.new(type: :multi_delegator, outputs: outputs) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-logger
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.22.0
 platform: ruby
 authors:
 - David Butler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-21 00:00:00.000000000 Z
+date: 2017-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-event
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -167,6 +181,7 @@ files:
 - lib/logstash-logger/device/file.rb
 - lib/logstash-logger/device/io.rb
 - lib/logstash-logger/device/kafka.rb
+- lib/logstash-logger/device/kinesis.rb
 - lib/logstash-logger/device/multi_delegator.rb
 - lib/logstash-logger/device/redis.rb
 - lib/logstash-logger/device/socket.rb
@@ -205,6 +220,7 @@ files:
 - spec/device/file_spec.rb
 - spec/device/io_spec.rb
 - spec/device/kafka_spec.rb
+- spec/device/kinesis_spec.rb
 - spec/device/multi_delegator_spec.rb
 - spec/device/redis_spec.rb
 - spec/device/socket_spec.rb
@@ -260,6 +276,7 @@ test_files:
 - spec/device/file_spec.rb
 - spec/device/io_spec.rb
 - spec/device/kafka_spec.rb
+- spec/device/kinesis_spec.rb
 - spec/device/multi_delegator_spec.rb
 - spec/device/redis_spec.rb
 - spec/device/socket_spec.rb