logstash-output-kinesis 1.5.1-java → 1.6.0-java

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDE4OTkzNDRlNzYzN2FiYThkMDZkYTgwZWViYTdmY2UxZDkzN2Y5ZQ==
+    NjhjNTE1ZmU1ZmFhNjQ5MmFhYzU4YmRlNDlmOTRlMmQxYmM2MDBjYQ==
   data.tar.gz: !binary |-
-    MWFmYjg5NjdiNGMyOTEzODIzNTExMjAzNGYwOGY5MTE0OGJkMGI4Yg==
+    OTc0YzRmMzM0YzZiMDJlNDFhM2UwMTMzOGE2ODk5ODg4MzM2MDNmMQ==
 SHA512:
   metadata.gz: !binary |-
-    ODNjZmFhMjllNjhmMjI2MDcwNDNiZGUxY2JiNjc1YWMzNTI1YzdlNzhjODg5
-    MzgyYzdhYjllNzNlMzQ5OGY1ZjViOGVhMzczZWE4YjQ5NDI1ZjMyYTU0MDYx
-    OGFlNTI0YTFlZjkwODFkMzM1NDQ4YjkzNjAyMGQzMTYyYzRiNWU=
+    M2VjMjNjMWFjMTJjOTBkNDgzYTQyYTE4YjAwM2I5MjhiMGNkODdjOWRmMTRi
+    ZmFlZGRjODBlNTk2NmIxMTBjMDYzZjg1NTliODhlMjkwODRhZjI4MmU2N2My
+    MTg5YjMwZWQ1NzRlYmE4ZDJmZmNhMDExOTZkZmYyNDg5NjdlZjY=
   data.tar.gz: !binary |-
-    MzhlNWFhYzk3MGQyNjc3YjQ3OGEwYWNkYmUyMGFiNjM4MGM1MWY1Nzk3OGQ5
-    OTdhMTliNzRlYTE3MzUxMzBkM2I0MDlmMTYzYzk0ZWU1ZGQ0YWE0NDhjMGFj
-    MWExMGNiZGM2MzMwMDE0ZTc5YWNmOWY4ZGVlNTMzMTg2MzBlZjA=
+    NDU0ZGFlZjE4YzJjZjBiNGQzOTAyYWFmZDEwM2RhMTA0YmVlMzg5NGE4Mjkx
+    MWYzZWU4MTFhYWU0YjNjZTlmYzVjZTk2Mzc1NmZmNjljYWQzMWFkNmEzMDg0
+    MTA0MzhlNGUxN2MxYmFkNTUwM2ViNDc0NmI2NmY4N2ZhOGJkMjM=

data/README.md

@@ -132,6 +132,19 @@ output {
 }
 ```
 
+### Backpressure
+
+The KPL library does not force any backpressure. This means if Kinesis is unavailable or throttling, KPL will happily accept records until it chews up all available memory on your machine. This plugin has a default backpressure mechanism - if there's more than 1000 pending records to be written to Kinesis, then further log records will block. This will cause Logstash to block further processing until everything is flushed out to Kinesis. I know that sounds lame, but it's better than the Linux OOM killer stepping in and breaking all your shit, no?
+
+Anyway, if you want to throw more memory / CPU cycles at buffering lots of stuff before it makes it to Kinesis, you can control the high-watermark:
+
+```nginx
+output {
+  kinesis {
+    max_pending_records => 10000 # I sure as hell hope you know what you're doing.
+  }
+}
+```
 
 ## Known Issues
 

data/lib/logstash-output-kinesis/version.rb

@@ -1,3 +1,3 @@
 module LogstashOutputKinesis
-  VERSION = "1.5.1"
+  VERSION = "1.6.0"
 end

data/lib/logstash/outputs/kinesis.rb

@@ -20,6 +20,9 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   config :event_partition_keys, :validate => :array, :default => []
   # If true, a random partition key will be assigned to each log record
   config :randomized_partition_key, :validate => :boolean, :default => false
+  # If the number of records pending being written to Kinesis exceeds this number, then block
+  # Logstash processing until they're all written.
+  config :max_pending_records, :validate => :number, :default => 1000
 
   # An AWS access key to use for authentication to Kinesis and CloudWatch
   config :access_key, :validate => :string
@@ -191,6 +194,13 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
     rescue => e
       @logger.warn("Error writing event to Kinesis", :exception => e)
     end
+
+    num = @producer.getOutstandingRecordsCount()
+    if num > @max_pending_records
+      @logger.warn("Kinesis is too busy - blocking until things have cleared up")
+      @producer.flushSync()
+      @logger.info("Okay - I've stopped blocking now")
+    end
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.0
 platform: java
 authors:
 - Sam Day
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-06 00:00:00.000000000 Z
+date: 2015-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core