logstash-output-kinesis 2.0.0-java → 2.0.1-java

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDBjODYwYzVmN2E0OTYxMWUzMTdhZWU2YjZlZmI3YWRkNzIxZWYwMg==
+    NWYxZmNmMWM3MjYxNjczM2JiMjRjZTQ3MDZlOTJkMWU1MzcyYzVmMw==
   data.tar.gz: !binary |-
-    Njc5MmRhOWNhMThiYWE1NTZlOTdlNGI4ZTFmOGM5YTc3YzViMmIxNA==
+    NzczNzJlMzQ3NTU2Y2YxNTI4MDY5OWU3YWVlY2QyYmYxZjQ3OTZjOQ==
 SHA512:
   metadata.gz: !binary |-
-    NjQ0YTMwMGM1NWQ1OWUwNDRhMTVhZWFmNjA5NDAxMWNkNWExMDVjZTdiOGM1
-    MjRjMGZjNmYyNjY2Mzk3ZDZiMmUwY2MyMTlhODIxNDZhNDY4OWQwMmY4NWIy
-    ZWE1YTVmYjNkMDk2MTAzZGQ0NWJmYTE4ZTM2YTIxNThjMzE4Nzk=
+    NDUwNTBjZDFjNmM5NjI3NjBhZjRmYWUxMDVmMWExMWY2ODAxZDc2ODljYzBl
+    MzA0Njc5Mzc4NDY5ZTI3MjFiOTU1ODRkMDBiODNkYTFiYzFhMjZhNjEzNGMy
+    NTMyZWU4MTU1MzQxMjE3NWVhMjgyNTQ0YmUwNjFhZGQ4ODk4Yzc=
   data.tar.gz: !binary |-
-    YjMxMjMwOTc4MTkzMjNjYTI0OTFmZGY3YThlYTNlNjMzNjA2MGI5ZGVjNTY4
-    ZjRmZDE0ZmFlNTMwZGZkZWFlNGMxZDU0MDI5ZDg1MTRhNWYzOTc5MWFmNTEy
-    YTQ3YWU0MDFkODIxNjg3MmNkMjE0ODllNDRhN2RjNWY4ZDczMGI=
+    NWE4NDA2NzZkN2NjNTUzMzc3MmE2NDJhMTNmZmFhZmY3MGE4ZGJlNGU4MjBm
+    MTM2ODE0Mzg1NGY1OGMxZDcxMzE5YWU0OWM4NWZkMTMyNDE3NmFlZGUzMjM5
+    NTA2ODYzNWExZmFhMWZlMWU2NGMxMzVhZjhmNzZhZjFhMWY2ZWY=

data/README.md

@@ -104,6 +104,18 @@ output {
 }
 ```
 
+#### Randomised partition keys
+
+If you don't care about the ordering of your logs in the Kinesis stream, you might want to use a random partition key. This way, your log stream will be more or less uniformly spread across all available shards in the Kinesis stream.
+
+```nginx
+output {
+  kinesis {
+    randomized_partition_key => true
+  }
+}
+```
+
 ### Record Aggregation
 
 The [Amazon KPL library can aggregate](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-concepts.html#d0e3423) your records when writing to the Kinesis stream. **This behaviour is configured to be enabled by default.**

data/lib/logstash-output-kinesis/version.rb

@@ -1,3 +1,3 @@
 module LogstashOutputKinesis
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/lib/logstash/outputs/kinesis.rb

@@ -3,6 +3,7 @@
 require "java"
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "securerandom"
 require "logstash-output-kinesis_jars"
 
 # Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
@@ -17,6 +18,8 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   config :stream_name, :validate => :string, :required => true
   # A list of event data keys to use when constructing a partition key
   config :event_partition_keys, :validate => :array, :default => []
+  # If true, a random partition key will be assigned to each log record
+  config :randomized_partition_key, :validate => :boolean, :default => false
 
   # An AWS access key to use for authentication to Kinesis and CloudWatch
   config :access_key, :validate => :string
@@ -78,18 +81,22 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def receive(event)
     return unless output?(event)
 
-    # Haha - gawd. If I don't put an empty string in the array, then calling .join()
-    # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
-    partition_key_parts = [""]
-
-    @event_partition_keys.each do |partition_key_name|
-      if not event[partition_key_name].nil? and event[partition_key_name].length > 0
-        partition_key_parts << event[partition_key_name].to_s
-        break
+    if @randomized_partition_key
+      event["[@metadata][partition_key]"] = SecureRandom.uuid
+    else
+      # Haha - gawd. If I don't put an empty string in the array, then calling .join()
+      # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
+      partition_key_parts = [""]
+
+      @event_partition_keys.each do |partition_key_name|
+        if not event[partition_key_name].nil? and event[partition_key_name].length > 0
+          partition_key_parts << event[partition_key_name].to_s
+          break
+        end
       end
-    end
 
-    event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+      event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+    end
 
     begin
       @codec.encode(event)

data/logstash-output-kinesis.gemspec

@@ -32,4 +32,9 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-json", "< 3.0.0"
   s.add_development_dependency "logstash-devutils"
   s.add_development_dependency "gem-release", "~>0.7.3"
+
+  # Temporary hack because Logstash devs are crazy.
+  # See: https://github.com/elastic/logstash/issues/4141
+  # We should be able to remove this once logstash-core >2.0.0 && >1.5.5 are released
+  s.add_development_dependency "concurrent-ruby", "0.9.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kinesis
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: java
 authors:
 - Sam Day
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-31 00:00:00.000000000 Z
+date: 2015-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -86,6 +86,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.3
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.9.1
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
   This gem is not a stand-alone program