logstash-input-kinesis 2.0.8-java → 2.0.10-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c30c3e57fd6684ef7828d46cae719101d821da13610d59b14943c4a98781968
-  data.tar.gz: 98599b307cf033adbc3f295182e7ef48780f0d933e33d3c803b341f2581f083d
+  metadata.gz: e043e41cda349ab19d8a7d315d12053e85720655957ecdd2bfa14c8004b3f251
+  data.tar.gz: 7f45606c39c9afe600125b147426fc6187ffccff94a8b202cda6c4e51391393e
 SHA512:
-  metadata.gz: d40c14d095aaab40a03d0234cf40ace49426346833cd9937312e501b06adfa523e214594d67fd5e8891c785c3b2c545807c240062e0d4f7e541301e287dedf0a
-  data.tar.gz: e0a3251ecdacaf66147313c7e518670804c8465db5036b787df9ec613a7ee4b3878d8210ff48546e487ec785f3a9191b5e6402b318372f171bd68fb9680ffe8f
+  metadata.gz: 2a49ae6fee952b540b522dca5cbab79cbdaabf68c2e9e628f1f88fbeea3b89f0bf4f03e8dfea65284d9125d68365e702536ee096dc161bcedb8eb9fe15120ba3
+  data.tar.gz: 67fa5c368c1924e31d3a47ad4082d06c957ade7c6d36e7d8023e54cd06a113bf1f2bd4de84d426bb68911f3a1f36e7eb5dc1ff73bb14cf5803554b7a5c621b21

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.0.10
+  - Added the ability to set additional settings exposed through KinesisClientLibConfiguration [#51](https://github.com/logstash-plugins/logstash-input-kinesis/pull/51)
+
+## 2.0.9
+  - Changed the 'workerid' to also include the host of the Logstash node [#48](https://github.com/logstash-plugins/logstash-input-kinesis/pull/48)
+
 ## 2.0.8
   - Changed plugin to use more recent versions of Kinesis Client library and AWS SDK[#45](https://github.com/logstash-plugins/logstash-input-kinesis/pull/45)
 

data/README.md

@@ -53,6 +53,11 @@ This are the properties you can configure and what are the default values:
     * **required**: false
     * **default value**: `"TRIM_HORIZON"`
 
+### Additional KCL Settings
+* `additional_settings`: The KCL provides several configuration options which can be set in [KinesisClientLibConfiguration](https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/coordinator/KinesisClientLibConfiguration.java). These options are configured via various function calls that all begin with `with`. Some of these functions take complex types, which are not supported. However, you may invoke any one of the `withX()` functions that take a primitive by providing key-value pairs in `snake_case`. For example, to set the dynamodb read and write capacity values, two functions exist, withInitialLeaseTableReadCapacity and withInitialLeaseTableWriteCapacity. To set a value for these, provide a hash of `additional_settings => {"initial_lease_table_read_capacity" => 25, "initial_lease_table_write_capacity" => 100}`
+    * **required**: false
+    * **default value**: `{}`
+
 ## Authentication
 
 This plugin uses the default AWS SDK auth chain, [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html), to determine which credentials the client will use, unless `profile` is set, in which case [ProfileCredentialsProvider](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/profile/ProfileCredentialsProvider.html) is used.

data/lib/logstash/inputs/kinesis.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+
+require "socket"
 require "logstash/inputs/base"
 require "logstash/errors"
 require "logstash/environment"
@@ -56,6 +58,9 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   # Select initial_position_in_stream. Accepts TRIM_HORIZON or LATEST
   config :initial_position_in_stream, :validate => ["TRIM_HORIZON", "LATEST"], :default => "TRIM_HORIZON"
 
+  # Any additional arbitrary kcl options configurable in the KinesisClientLibConfiguration
+  config :additional_settings, :validate => :hash, :default => {}
+
   def initialize(params = {})
     super(params)
   end
@@ -69,7 +74,9 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
       kinesis_logger.setLevel(org.apache.log4j::Level::WARN)
     end
 
-    worker_id = java.util::UUID.randomUUID.to_s
+    hostname = Socket.gethostname
+    uuid = java.util::UUID.randomUUID.to_s
+    worker_id = "#{hostname}:#{uuid}"
 
     # If the AWS profile is set, use the profile credentials provider.
     # Otherwise fall back to the default chain.
@@ -91,6 +98,13 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
       worker_id).
         withInitialPositionInStream(initial_position_in_stream).
         withRegionName(@region)
+
+      # Call arbitrary "withX()" functions
+      # snake_case => withCamelCase happens automatically
+      @additional_settings.each do |key, value|
+          fn = "with_#{key}"
+          @kcl_config.send(fn, value)
+      end
   end
 
   def run(output_queue)

data/lib/logstash/inputs/kinesis/version.rb

@@ -2,7 +2,7 @@
 module Logstash
   module Input
     module Kinesis
-      VERSION = "2.0.8"
+      VERSION = "2.0.10"
     end
   end
 end

data/spec/inputs/kinesis_spec.rb

@@ -38,6 +38,50 @@ RSpec.describe "inputs/kinesis" do
     "initial_position_in_stream" => "LATEST"
   }}
 
+  # Config hash to test valid additional_settings
+  let(:config_with_valid_additional_settings) {{
+    "application_name" => "my-processor",
+    "kinesis_stream_name" => "run-specs",
+    "codec" => codec,
+    "metrics" => metrics,
+    "checkpoint_interval_seconds" => 120,
+    "region" => "ap-southeast-1",
+    "profile" => nil,
+    "additional_settings" => {
+        "initial_lease_table_read_capacity" => 25,
+        "initial_lease_table_write_capacity" => 100,
+        "kinesis_endpoint" => "http://localhost"
+    }
+  }}
+
+  # Config hash to test invalid additional_settings where the name is not found
+  let(:config_with_invalid_additional_settings_name_not_found) {{
+    "application_name" => "my-processor",
+    "kinesis_stream_name" => "run-specs",
+    "codec" => codec,
+    "metrics" => metrics,
+    "checkpoint_interval_seconds" => 120,
+    "region" => "ap-southeast-1",
+    "profile" => nil,
+    "additional_settings" => {
+        "foo" => "bar"
+    }
+  }}
+
+  # Config hash to test invalid additional_settings where the type is complex or wrong
+  let(:config_with_invalid_additional_settings_wrong_type) {{
+    "application_name" => "my-processor",
+    "kinesis_stream_name" => "run-specs",
+    "codec" => codec,
+    "metrics" => metrics,
+    "checkpoint_interval_seconds" => 120,
+    "region" => "ap-southeast-1",
+    "profile" => nil,
+    "additional_settings" => {
+        "metrics_level" => "invalid_metrics_level"
+    }
+  }}
+
   subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config) }
   let(:kcl_worker) { double('kcl_worker') }
   let(:stub_builder) { double('kcl_builder', build: kcl_worker) }
@@ -77,6 +121,33 @@ RSpec.describe "inputs/kinesis" do
     expect(kinesis_with_latest.kcl_config.get_kinesis_credentials_provider.getClass.to_s).to eq("com.amazonaws.auth.DefaultAWSCredentialsProviderChain")
   end
 
+  subject!(:kinesis_with_valid_additional_settings) { LogStash::Inputs::Kinesis.new(config_with_valid_additional_settings) }
+
+  it "configures the KCL" do
+    kinesis_with_valid_additional_settings.register
+    expect(kinesis_with_valid_additional_settings.kcl_config.applicationName).to eq("my-processor")
+    expect(kinesis_with_valid_additional_settings.kcl_config.streamName).to eq("run-specs")
+    expect(kinesis_with_valid_additional_settings.kcl_config.regionName).to eq("ap-southeast-1")
+    expect(kinesis_with_valid_additional_settings.kcl_config.initialLeaseTableReadCapacity).to eq(25)
+    expect(kinesis_with_valid_additional_settings.kcl_config.initialLeaseTableWriteCapacity).to eq(100)
+    expect(kinesis_with_valid_additional_settings.kcl_config.kinesisEndpoint).to eq("http://localhost")
+  end
+
+
+  subject!(:kinesis_with_invalid_additional_settings_name_not_found) { LogStash::Inputs::Kinesis.new(config_with_invalid_additional_settings_name_not_found) }
+
+  it "raises NoMethodError for invalid configuration options" do
+    expect{ kinesis_with_invalid_additional_settings_name_not_found.register }.to raise_error(NoMethodError)
+  end
+
+
+  subject!(:kinesis_with_invalid_additional_settings_wrong_type) { LogStash::Inputs::Kinesis.new(config_with_invalid_additional_settings_wrong_type) }
+
+  it "raises an error for invalid configuration values such as the wrong type" do
+    expect{ kinesis_with_invalid_additional_settings_wrong_type.register }.to raise_error(Java::JavaLang::IllegalArgumentException)
+  end
+
+
   context "#run" do
     it "runs the KCL worker" do
       expect(kinesis).to receive(:kcl_builder).with(queue).and_return(stub_builder)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kinesis
 version: !ruby/object:Gem::Version
-  version: 2.0.8
+  version: 2.0.10
 platform: java
 authors:
 - Brian Palmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-25 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement