logstash-output-kinesis 1.5.0-java → 1.5.1-java

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZWNiNzM3ODE5ZDc5MDRkNWJmMzkyOTNmNmY2YjczZjIwOGZkMGQ2MA==
+    NDE4OTkzNDRlNzYzN2FiYThkMDZkYTgwZWViYTdmY2UxZDkzN2Y5ZQ==
   data.tar.gz: !binary |-
-    OGUxZGY5OGQ4ZjI4OTJjNTMzNzI2ZDkyMjJiYjhhYzExNTE1YTlmYw==
+    MWFmYjg5NjdiNGMyOTEzODIzNTExMjAzNGYwOGY5MTE0OGJkMGI4Yg==
 SHA512:
   metadata.gz: !binary |-
-    ZmVlNjQyYmRkMGU0ZDcyYWQ4MjIwODNjZGIyZmQ3Zjg1YTM4NWM3M2ZiNTNj
-    Y2NkMmU3MjFlMTI0NGIxZGY5MjIwMjU0NDhjYWEzZGFjMjUzNTk2ZTJjMWFi
-    NzkxNjBlMDgyMTFlOWM0ZTA2MTRhYTYxYzk4ZmM5YjQ0ZjljMzU=
+    ODNjZmFhMjllNjhmMjI2MDcwNDNiZGUxY2JiNjc1YWMzNTI1YzdlNzhjODg5
+    MzgyYzdhYjllNzNlMzQ5OGY1ZjViOGVhMzczZWE4YjQ5NDI1ZjMyYTU0MDYx
+    OGFlNTI0YTFlZjkwODFkMzM1NDQ4YjkzNjAyMGQzMTYyYzRiNWU=
   data.tar.gz: !binary |-
-    ZTQyOTU2ZTQ4NTliM2JiODM0ODA0YmUwYTIzMjdmNWIwYmMwOWI5M2RjYzRj
-    N2U4NTA0NTgzNTM1ZjkyMjBjMDhjNWNmNTY4Njg4ZDBhYmM3YTMzMmE1Yzc5
-    NjE0ZWVhNzZiMDVmZTI2NDQyODY2MTQwZjBiMDM4N2QwMTc0ZGE=
+    MzhlNWFhYzk3MGQyNjc3YjQ3OGEwYWNkYmUyMGFiNjM4MGM1MWY1Nzk3OGQ5
+    OTdhMTliNzRlYTE3MzUxMzBkM2I0MDlmMTYzYzk0ZWU1ZGQ0YWE0NDhjMGFj
+    MWExMGNiZGM2MzMwMDE0ZTc5YWNmOWY4ZGVlNTMzMTg2MzBlZjA=

data/README.md

@@ -104,6 +104,18 @@ output {
 }
 ```
 
+#### Randomised partition keys
+
+If you don't care about the ordering of your logs in the Kinesis stream, you might want to use a random partition key. This way, your log stream will be more or less uniformly spread across all available shards in the Kinesis stream.
+
+```nginx
+output {
+  kinesis {
+    randomized_partition_key => true
+  }
+}
+```
+
 ### Record Aggregation
 
 The [Amazon KPL library can aggregate](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-concepts.html#d0e3423) your records when writing to the Kinesis stream. **This behaviour is configured to be enabled by default.**

data/lib/logstash-output-kinesis/version.rb

@@ -1,3 +1,3 @@
 module LogstashOutputKinesis
-  VERSION = "1.5.0"
+  VERSION = "1.5.1"
 end

data/lib/logstash/outputs/kinesis.rb

@@ -3,6 +3,7 @@
 require "java"
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "securerandom"
 require "logstash-output-kinesis_jars"
 
 # Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
@@ -17,6 +18,8 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   config :stream_name, :validate => :string, :required => true
   # A list of event data keys to use when constructing a partition key
   config :event_partition_keys, :validate => :array, :default => []
+  # If true, a random partition key will be assigned to each log record
+  config :randomized_partition_key, :validate => :boolean, :default => false
 
   # An AWS access key to use for authentication to Kinesis and CloudWatch
   config :access_key, :validate => :string
@@ -78,18 +81,22 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def receive(event)
     return unless output?(event)
 
-    # Haha - gawd. If I don't put an empty string in the array, then calling .join()
-    # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
-    partition_key_parts = [""]
-
-    @event_partition_keys.each do |partition_key_name|
-      if not event[partition_key_name].nil? and event[partition_key_name].length > 0
-        partition_key_parts << event[partition_key_name].to_s
-        break
+    if @randomized_partition_key
+      event["[@metadata][partition_key]"] = SecureRandom.uuid
+    else
+      # Haha - gawd. If I don't put an empty string in the array, then calling .join()
+      # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
+      partition_key_parts = [""]
+
+      @event_partition_keys.each do |partition_key_name|
+        if not event[partition_key_name].nil? and event[partition_key_name].length > 0
+          partition_key_parts << event[partition_key_name].to_s
+          break
+        end
       end
-    end
 
-    event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+      event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+    end
 
     begin
       @codec.encode(event)

data/logstash-output-kinesis.gemspec

@@ -32,4 +32,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-json", "< 2.0.0"
   s.add_development_dependency "logstash-devutils", "< 2.0.0"
   s.add_development_dependency "gem-release", "~>0.7.3"
+
+  # Temporary hack because Logstash devs are crazy.
+  # See: https://github.com/elastic/logstash/issues/4141
+  # We should be able to remove this once logstash-core >2.0.0 && >1.5.5 are released
+  s.add_development_dependency "concurrent-ruby", "0.9.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.1
 platform: java
 authors:
 - Sam Day
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-31 00:00:00.000000000 Z
+date: 2015-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -86,6 +86,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.3
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
   This gem is not a stand-alone program