logstash-output-kinesis 1.6.0-java → 2.0.0-java

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjhjNTE1ZmU1ZmFhNjQ5MmFhYzU4YmRlNDlmOTRlMmQxYmM2MDBjYQ==
+    NDBjODYwYzVmN2E0OTYxMWUzMTdhZWU2YjZlZmI3YWRkNzIxZWYwMg==
   data.tar.gz: !binary |-
-    OTc0YzRmMzM0YzZiMDJlNDFhM2UwMTMzOGE2ODk5ODg4MzM2MDNmMQ==
+    Njc5MmRhOWNhMThiYWE1NTZlOTdlNGI4ZTFmOGM5YTc3YzViMmIxNA==
 SHA512:
   metadata.gz: !binary |-
-    M2VjMjNjMWFjMTJjOTBkNDgzYTQyYTE4YjAwM2I5MjhiMGNkODdjOWRmMTRi
-    ZmFlZGRjODBlNTk2NmIxMTBjMDYzZjg1NTliODhlMjkwODRhZjI4MmU2N2My
-    MTg5YjMwZWQ1NzRlYmE4ZDJmZmNhMDExOTZkZmYyNDg5NjdlZjY=
+    NjQ0YTMwMGM1NWQ1OWUwNDRhMTVhZWFmNjA5NDAxMWNkNWExMDVjZTdiOGM1
+    MjRjMGZjNmYyNjY2Mzk3ZDZiMmUwY2MyMTlhODIxNDZhNDY4OWQwMmY4NWIy
+    ZWE1YTVmYjNkMDk2MTAzZGQ0NWJmYTE4ZTM2YTIxNThjMzE4Nzk=
   data.tar.gz: !binary |-
-    NDU0ZGFlZjE4YzJjZjBiNGQzOTAyYWFmZDEwM2RhMTA0YmVlMzg5NGE4Mjkx
-    MWYzZWU4MTFhYWU0YjNjZTlmYzVjZTk2Mzc1NmZmNjljYWQzMWFkNmEzMDg0
-    MTA0MzhlNGUxN2MxYmFkNTUwM2ViNDc0NmI2NmY4N2ZhOGJkMjM=
+    YjMxMjMwOTc4MTkzMjNjYTI0OTFmZGY3YThlYTNlNjMzNjA2MGI5ZGVjNTY4
+    ZjRmZDE0ZmFlNTMwZGZkZWFlNGMxZDU0MDI5ZDg1MTRhNWYzOTc5MWFmNTEy
+    YTQ3YWU0MDFkODIxNjg3MmNkMjE0ODllNDRhN2RjNWY4ZDczMGI=

data/README.md

@@ -1,11 +1,11 @@
-# Kinesis Output Plugin `1.5.x`
+# Kinesis Output Plugin
 
 [![Build Status][badge-travis]][travis]
 [![Gem info][badge-gem]][rubygems]
 
 This is a plugin for [Logstash](https://github.com/elasticsearch/logstash). It will send log records to a [Kinesis stream](https://aws.amazon.com/kinesis/), using the [Kinesis Producer Library (KPL)](https://docs.aws.amazon.com/kinesis/latest/dev/developing-producers-with-kpl.html).
 
-**This version is intended for use with Logstash 1.5.x. Please use a 2.x version of this plugin for Logstash 2.0+ compatbility.**
+**This version is intended for use with Logstash 2.x. Please use a [1.5.x](https://github.com/samcday/logstash-output-kinesis/tree/1.5) version of this plugin for Logstash 1.5.x compatibility.**
 
 
 ## Configuration
@@ -104,18 +104,6 @@ output {
 }
 ```
 
-#### Randomised partition keys
-
-If you don't care about the ordering of your logs in the Kinesis stream, you might want to use a random partition key. This way, your log stream will be more or less uniformly spread across all available shards in the Kinesis stream.
-
-```nginx
-output {
-  kinesis {
-    randomized_partition_key => true
-  }
-}
-```
-
 ### Record Aggregation
 
 The [Amazon KPL library can aggregate](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-concepts.html#d0e3423) your records when writing to the Kinesis stream. **This behaviour is configured to be enabled by default.**
@@ -132,19 +120,6 @@ output {
 }
 ```
 
-### Backpressure
-
-The KPL library does not force any backpressure. This means if Kinesis is unavailable or throttling, KPL will happily accept records until it chews up all available memory on your machine. This plugin has a default backpressure mechanism - if there's more than 1000 pending records to be written to Kinesis, then further log records will block. This will cause Logstash to block further processing until everything is flushed out to Kinesis. I know that sounds lame, but it's better than the Linux OOM killer stepping in and breaking all your shit, no?
-
-Anyway, if you want to throw more memory / CPU cycles at buffering lots of stuff before it makes it to Kinesis, you can control the high-watermark:
-
-```nginx
-output {
-  kinesis {
-    max_pending_records => 10000 # I sure as hell hope you know what you're doing.
-  }
-}
-```
 
 ## Known Issues
 

data/lib/logstash-output-kinesis/version.rb

@@ -1,3 +1,3 @@
 module LogstashOutputKinesis
-  VERSION = "1.6.0"
+  VERSION = "2.0.0"
 end

data/lib/logstash/outputs/kinesis.rb

@@ -3,7 +3,6 @@
 require "java"
 require "logstash/outputs/base"
 require "logstash/namespace"
-require "securerandom"
 require "logstash-output-kinesis_jars"
 
 # Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
@@ -18,11 +17,6 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   config :stream_name, :validate => :string, :required => true
   # A list of event data keys to use when constructing a partition key
   config :event_partition_keys, :validate => :array, :default => []
-  # If true, a random partition key will be assigned to each log record
-  config :randomized_partition_key, :validate => :boolean, :default => false
-  # If the number of records pending being written to Kinesis exceeds this number, then block
-  # Logstash processing until they're all written.
-  config :max_pending_records, :validate => :number, :default => 1000
 
   # An AWS access key to use for authentication to Kinesis and CloudWatch
   config :access_key, :validate => :string
@@ -84,23 +78,19 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def receive(event)
     return unless output?(event)
 
-    if @randomized_partition_key
-      event["[@metadata][partition_key]"] = SecureRandom.uuid
-    else
-      # Haha - gawd. If I don't put an empty string in the array, then calling .join()
-      # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
-      partition_key_parts = [""]
-
-      @event_partition_keys.each do |partition_key_name|
-        if not event[partition_key_name].nil? and event[partition_key_name].length > 0
-          partition_key_parts << event[partition_key_name].to_s
-          break
-        end
-      end
+    # Haha - gawd. If I don't put an empty string in the array, then calling .join()
+    # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
+    partition_key_parts = [""]
 
-      event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+    @event_partition_keys.each do |partition_key_name|
+      if not event[partition_key_name].nil? and event[partition_key_name].length > 0
+        partition_key_parts << event[partition_key_name].to_s
+        break
+      end
     end
 
+    event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+
     begin
       @codec.encode(event)
     rescue => e
@@ -109,10 +99,9 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   end
 
   public
-  def teardown
+  def close
     @producer.flushSync()
     @producer.destroy()
-    finished()
   end
 
   def create_kpl_config
@@ -194,13 +183,6 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
     rescue => e
       @logger.warn("Error writing event to Kinesis", :exception => e)
     end
-
-    num = @producer.getOutstandingRecordsCount()
-    if num > @max_pending_records
-      @logger.warn("Kinesis is too busy - blocking until things have cleared up")
-      @producer.flushSync()
-      @logger.info("Okay - I've stopped blocking now")
-    end
   end
 end
 

data/logstash-output-kinesis.gemspec

@@ -27,14 +27,9 @@ Gem::Specification.new do |s|
   s.platform = "java"
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
-  s.add_runtime_dependency "logstash-codec-plain", "< 2.0.0"
-  s.add_runtime_dependency "logstash-codec-json", "< 2.0.0"
-  s.add_development_dependency "logstash-devutils", "< 2.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-plain", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-json", "< 3.0.0"
+  s.add_development_dependency "logstash-devutils"
   s.add_development_dependency "gem-release", "~>0.7.3"
-
-  # Temporary hack because Logstash devs are crazy.
-  # See: https://github.com/elastic/logstash/issues/4141
-  # We should be able to remove this once logstash-core >2.0.0 && >1.5.5 are released
-  s.add_development_dependency "concurrent-ruby", "0.9.1"
 end

data/spec/outputs/kinesis_spec.rb

@@ -21,7 +21,7 @@ describe LogStash::Outputs::Kinesis do
       output = LogStash::Outputs::Kinesis.new (config)
       output.register
       output.receive(sample_event)
-      output.teardown
+      output.close
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: java
 authors:
 - Sam Day
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-05 00:00:00.000000000 Z
+date: 2015-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -16,62 +16,62 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: gem-release
   requirement: !ruby/object:Gem::Requirement
@@ -86,20 +86,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.3
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.9.1
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
   This gem is not a stand-alone program