logstash-input-kinesis 1.3.1-java → 1.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 83fd102abe45c289fa1a52e9bff99ce35d6fc07b
-  data.tar.gz: ab03d504c9c2e4448425fdad4f41218214960a63
+  metadata.gz: e0660cef254c9970b671583df6a7c95489ae888f
+  data.tar.gz: d1a24cc652091346a8f0102ac532bf462f00b310
 SHA512:
-  metadata.gz: 2269a2acf2d07448b222a9020044b949166286a68b0683b72abecabfb780d6d37e71e50d3e87de74cc17cb2da33782af5dfee5a8e4f35f7fc844a8d8434c402c
-  data.tar.gz: 22470cf8c07ef38db0b2036853732ca527212c5abe03e2613021a497e1f320be534272ddafcb8b3c9c740005bf42a18b3edd22c63b7bcb026205865ee392ea4d
+  metadata.gz: 945f30cbd64354c94addfbeb96be414c8a9d1fb0bb96efa70fac0b403242ddcd04c44864d396d7859a35395ca92861e3e7453590156acdb20688a7c03202288b
+  data.tar.gz: e3ccec93560deb6bd871517cc6238c663e49ac9e63fb519757c0c1c70389f3a3732944c858c8a0f9483af87086fc170372911e39f3377c2fbad30ab22afab5d5

data/README.md

@@ -1,9 +1,9 @@
 # Logstash AWS Kinesis Input Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
-
 [![Build Status](https://travis-ci.org/codekitchen/logstash-input-kinesis.svg)](https://travis-ci.org/codekitchen/logstash-input-kinesis)
 
+This is a [AWS Kinesis](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html) input plugin for [Logstash](https://github.com/elasticsearch/logstash). Under the hood uses the [Kinesis Client Library](http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-implementation-app-java.html).
+
 ## Installation
 
 This plugin requires Logstash 1.5, and can be installed by Logstash
@@ -24,7 +24,40 @@ input {
 }
 ```
 
-At least `kinesis_stream_name` is requred.
+## Configuration
+
+This are the properties you can configure and what are the default values:
+
+* `application_name`: The name of the application used in DynamoDB for coordination. Only one worker per unique stream partition and application will be actively consuming messages.
+    * **required**: false
+    * **default value**: `logstash`
+* `kinesis_stream_name`: The Kinesis stream name.
+    * **required**: true
+* `region`: The AWS region name for Kinesis, DynamoDB and Cloudwatch (if enabled)
+    * **required**: false
+    * **default value**: `us-east-1`
+* `checkpoint_interval_seconds`: How many seconds between worker checkpoints to DynamoDB. A low value ussually means lower message replay in case of node failure/restart but it increases CPU+network ussage (which increases the AWS costs).
+    * **required**: false
+    * **default value**: `60`
+* `metrics`: Worker metric tracking. By default this is disabled, set it to "cloudwatch" to enable the cloudwatch integration in the Kinesis Client Library. 
+    * **required**: false
+    * **default value**: `nil`
+
+## Authentication
+
+This plugin uses the default AWS SDK auth chain, [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html), to determine which credentials the client will use.
+
+The default chain follows this order trying to read the credentials:
+ * `AWS_ACCESS_KEY_ID` / `AWS_SECRET_KEY` environment variables
+ * `~/.aws/credentials` credentials file
+ * EC2 instance profile
+
+The credentials will need access to the following services:
+* AWS Kinesis
+* AWS DynamoDB: the client library stores information for worker coordination in DynamoDB (offsets and active worker per partition)
+* AWS CloudWatch: if the metrics are enabled the credentials need CloudWatch update permisions granted.
+
+Look at the [documentation](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) for deeper information.
 
 ## Contributing
 

data/lib/logstash/inputs/kinesis.rb

@@ -22,6 +22,7 @@ require "logstash/inputs/kinesis/version"
 # The library can optionally also send worker statistics to CloudWatch.
 class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
+  KCL_PROCESSOR_FACTORY_CLASS = com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory
   require "logstash/inputs/kinesis/worker"
 
   config_name 'kinesis'
@@ -49,8 +50,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   # to enable the cloudwatch integration in the Kinesis Client Library.
   config :metrics, :validate => [nil, "cloudwatch"], :default => nil
 
-  def initialize(params = {}, kcl_class = KCL::Worker)
-    @kcl_class = kcl_class
+  def initialize(params = {})
     super(params)
   end
 
@@ -60,7 +60,7 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
       logger.setLevel(java.util.logging::Level::WARNING)
 
     worker_id = java.util::UUID.randomUUID.to_s
-    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new()
+    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
     @kcl_config = KCL::KinesisClientLibConfiguration.new(
       @application_name,
       @kinesis_stream_name,
@@ -71,25 +71,29 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
   end
 
   def run(output_queue)
-    worker_factory = proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
-    if metrics_factory
-      @kcl_worker = @kcl_class.new(
-        worker_factory,
-        @kcl_config,
-        metrics_factory)
-    else
-      @kcl_worker = @kcl_class.new(
-        worker_factory,
-        @kcl_config)
-    end
+    @kcl_worker = kcl_builder(output_queue).build
+    @kcl_worker.run
+  end
+
+  def kcl_builder(output_queue)
+    KCL::Worker::Builder.new.tap do |builder|
+      builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
+      builder.config(@kcl_config)
 
-    @kcl_worker.run()
+      if metrics_factory
+        builder.metricsFactory(metrics_factory)
+      end
+    end
   end
 
   def teardown
     @kcl_worker.shutdown if @kcl_worker
   end
 
+  def worker_factory(output_queue)
+    proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
+  end
+
   protected
 
   def metrics_factory

data/lib/logstash/inputs/kinesis/version.rb

@@ -1,7 +1,7 @@
 module Logstash
   module Input
     module Kinesis
-      VERSION = "1.3.1"
+      VERSION = "1.4.0"
     end
   end
 end

data/lib/logstash/inputs/kinesis/worker.rb

@@ -1,5 +1,5 @@
 class LogStash::Inputs::Kinesis::Worker
-  include com.amazonaws.services.kinesis.clientlibrary.interfaces::IRecordProcessor
+  include com.amazonaws.services.kinesis.clientlibrary.interfaces.v2::IRecordProcessor
 
   attr_reader(
     :checkpoint_interval,
@@ -16,23 +16,23 @@ class LogStash::Inputs::Kinesis::Worker
       @next_checkpoint = Time.now - 600
       @constructed = true
     else
-      _shard_id, _ = args
+      _shard_id = args[0].shardId
       @decoder = java.nio.charset::Charset.forName("UTF-8").newDecoder()
     end
   end
   public :initialize
 
-  def processRecords(records, checkpointer)
-    records.each { |record| process_record(record) }
+  def processRecords(records_input)
+    records_input.records.each { |record| process_record(record) }
     if Time.now >= @next_checkpoint
-      checkpoint(checkpointer)
+      checkpoint(records_input.checkpointer)
       @next_checkpoint = Time.now + @checkpoint_interval
     end
   end
 
-  def shutdown(checkpointer, reason)
-    if reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
-      checkpoint(checkpointer)
+  def shutdown(shutdown_input)
+    if shutdown_input.reason == com.amazonaws.services.kinesis.clientlibrary.types::ShutdownReason::TERMINATE
+      checkpoint(shutdown_input.checkpointer)
     end
   end
 

data/pom.xml

@@ -8,7 +8,7 @@
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>amazon-kinesis-client</artifactId>
-      <version>1.2.1</version>
+      <version>1.6.1</version>
     </dependency>
   </dependencies>
   <build>

data/spec/inputs/kinesis/worker_spec.rb

@@ -1,19 +1,22 @@
+require 'logstash-input-kinesis_jars'
 require "logstash/plugin"
 require "logstash/inputs/kinesis"
 require "logstash/codecs/json"
 require "json"
 
 RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
+  KCL_TYPES = com.amazonaws.services.kinesis.clientlibrary.types
+
   subject!(:worker) { LogStash::Inputs::Kinesis::Worker.new(codec, queue, decorator, checkpoint_interval) }
   let(:codec) { LogStash::Codecs::JSON.new() }
   let(:queue) { Queue.new }
   let(:decorator) { proc { |x| x["decorated"] = true; x } }
   let(:checkpoint_interval) { 120 }
   let(:checkpointer) { double('checkpointer', checkpoint: nil) }
-  let(:shard_id) { "xyz" }
+  let(:init_input) { KCL_TYPES::InitializationInput.new().withShardId("xyz") }
 
   it "honors the initialize java interface method contract" do
-    expect { worker.initialize(shard_id) }.to_not raise_error
+    expect { worker.initialize(init_input) }.to_not raise_error
   end
 
   def record(hash = { "message" => "test" })
@@ -22,17 +25,29 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
     double(getData: data)
   end
 
-  let(:record1) { record(id: "record1", message: "test1") }
-  let(:record2) { record(id: "record2", message: "test2") }
+  let(:process_input) { 
+    KCL_TYPES::ProcessRecordsInput.new()
+        .withRecords(java.util.Arrays.asList([
+            record(id: "record1", message: "test1"), 
+            record(id: "record2", message: "test2")
+          ].to_java)
+        )
+        .withCheckpointer(checkpointer) 
+  }
+  let(:empty_process_input) { 
+    KCL_TYPES::ProcessRecordsInput.new()
+        .withRecords(java.util.Arrays.asList([].to_java))
+        .withCheckpointer(checkpointer) 
+  }
 
   context "initialized" do
     before do
-      worker.initialize(shard_id)
+      worker.initialize(init_input)
     end
 
     describe "#processRecords" do
       it "decodes and queues each record with decoration" do
-        worker.processRecords([record1, record2], checkpointer)
+        worker.processRecords(process_input)
         m1 = queue.pop
         m2 = queue.pop
         expect(m1).to be_kind_of(LogStash::Event)
@@ -44,14 +59,14 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
 
       it "checkpoints on interval" do
         expect(checkpointer).to receive(:checkpoint).once
-        worker.processRecords([], checkpointer)
+        worker.processRecords(empty_process_input)
 
         # not this time
-        worker.processRecords([], checkpointer)
+        worker.processRecords(empty_process_input)
 
         allow(Time).to receive(:now).and_return(Time.now + 125)
         expect(checkpointer).to receive(:checkpoint).once
-        worker.processRecords([], checkpointer)
+        worker.processRecords(empty_process_input)
       end
     end
   end

data/spec/inputs/kinesis_spec.rb

@@ -2,17 +2,10 @@ require "logstash/plugin"
 require "logstash/inputs/kinesis"
 require "logstash/codecs/json"
 
+
 RSpec.describe "inputs/kinesis" do
   KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
 
-  # It's very difficult to directly test the Java class, with all its private members,
-  # so we subclass it here.
-  class TestKCLWorker < KCL::Worker
-    field_reader :metricsFactory, :recordProcessorFactory
-    def run
-    end
-  end
-
   let(:config) {{
     "application_name" => "my-processor",
     "kinesis_stream_name" => "run-specs",
@@ -22,10 +15,12 @@ RSpec.describe "inputs/kinesis" do
     "region" => "ap-southeast-1",
   }}
 
-  subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config, kcl_class) }
-  let(:kcl_class) { TestKCLWorker }
+  subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config) }
+  let(:kcl_worker) { double('kcl_worker') }
+  let(:stub_builder) { double('kcl_builder', build: kcl_worker) }
   let(:metrics) { nil }
   let(:codec) { LogStash::Codecs::JSON.new() }
+  let(:queue) { Queue.new }
 
   it "registers without error" do
     input = LogStash::Plugin.lookup("input", "kinesis").new("kinesis_stream_name" => "specs", "codec" => codec)
@@ -41,42 +36,63 @@ RSpec.describe "inputs/kinesis" do
   end
 
   context "#run" do
-    let(:queue) { Queue.new }
-
-    before do
-      kinesis.register
+    it "runs the KCL worker" do
+      expect(kinesis).to receive(:kcl_builder).with(queue).and_return(stub_builder)
+      expect(kcl_worker).to receive(:run).with(no_args)
+      builder = kinesis.run(queue)
     end
+  end
 
+  context "#worker_factory" do
     it "clones the codec for each worker" do
       expect(codec).to receive(:clone).once
-      kinesis.run(queue)
-      worker = kinesis.kcl_worker.recordProcessorFactory.call()
+      worker = kinesis.worker_factory(queue).call()
       expect(worker).to be_kind_of(LogStash::Inputs::Kinesis::Worker)
     end
 
-    it "generates a valid worker via the factory proc" do
-      kinesis.run(queue)
-      worker = kinesis.kcl_worker.recordProcessorFactory.call()
+    it "generates a valid worker" do
+      worker = kinesis.worker_factory(queue).call()
+
       expect(worker.codec).to be_kind_of(codec.class)
       expect(worker.checkpoint_interval).to eq(120)
       expect(worker.output_queue).to eq(queue)
       expect(worker.decorator).to eq(kinesis.method(:decorate))
       expect(worker.logger).to eq(kinesis.logger)
     end
+  end
+
+  # these tests are heavily dependent on the current Worker::Builder
+  # implementation because its state is all private
+  context "#kcl_builder" do
+    let(:builder) { kinesis.kcl_builder(queue) }
+
+    it "sets the worker factory" do
+      expect(field(builder, "recordProcessorFactory")).to_not eq(nil)
+    end
+
+    it "sets the config" do
+      kinesis.register
+      config = field(builder, "config")
+      expect(config).to eq(kinesis.kcl_config)
+    end
 
     it "disables metric tracking by default" do
-      kinesis.run(queue)
-      expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
+      expect(field(builder, "metricsFactory")).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
     end
 
     context "cloudwatch" do
       let(:metrics) { "cloudwatch" }
       it "uses cloudwatch metrics if specified" do
-        kinesis.run(queue)
-        expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::CWMetricsFactory)
-        # the process hangs otherwise
-        kinesis.kcl_worker.metricsFactory.shutdown
+        # since the behaviour is enclosed on private methods it is not testable. So here
+        # the expected value can be tested, not the result associated to set this value
+        expect(field(builder, "metricsFactory")).to eq(nil)
       end
     end
   end
+
+  def field(obj, name)
+    field = obj.java_class.declared_field(name)
+    field.accessible = true
+    field.value(obj)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: java
 authors:
 - Brian Palmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-17 00:00:00.000000000 Z
+date: 2015-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -103,60 +103,24 @@ files:
 - spec/inputs/kinesis/worker_spec.rb
 - spec/inputs/kinesis_spec.rb
 - spec/spec_helper.rb
-- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.2.1.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-autoscaling-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudformation-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudfront-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudhsm-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudsearch-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar
 - vendor/jar-dependencies/runtime-jars/commons-codec-1.6.jar
+- vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar
 - vendor/jar-dependencies/runtime-jars/commons-logging-1.1.3.jar
-- vendor/jar-dependencies/runtime-jars/httpclient-4.3.4.jar
-- vendor/jar-dependencies/runtime-jars/httpcore-4.3.2.jar
-- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.3.0.jar
-- vendor/jar-dependencies/runtime-jars/jackson-core-2.3.2.jar
-- vendor/jar-dependencies/runtime-jars/jackson-databind-2.3.2.jar
+- vendor/jar-dependencies/runtime-jars/guava-18.0.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.3.6.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-4.3.3.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar
 - vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
+- vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar
 homepage: https://github.com/codekitchen/logstash-input-kinesis
 licenses:
 - Apache License (2.0)