logstash-input-kinesis 1.3.0-java → 1.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c1b5d3fcaf40bd2b6c768604156b16b8a36013f
-  data.tar.gz: 329f83883fea7dd162e5f9c249485aca277c5f09
+  metadata.gz: 83fd102abe45c289fa1a52e9bff99ce35d6fc07b
+  data.tar.gz: ab03d504c9c2e4448425fdad4f41218214960a63
 SHA512:
-  metadata.gz: b3faf9e236029c99016c5e3de9e76f973bcc1f8fe13583043914518472bae424b99b40d1a85470f43a2fdb35c7ae052c5af0675100ba975379ac881610496d26
-  data.tar.gz: cd99d315da9c8a2783ca75d1480c8bf0aa7d8c794e5f7c4bf403c49d3d660133a7af2ae5abff831f69ef092e3d10782ef0cd86a6fc428997437a3669cd6a8628
+  metadata.gz: 2269a2acf2d07448b222a9020044b949166286a68b0683b72abecabfb780d6d37e71e50d3e87de74cc17cb2da33782af5dfee5a8e4f35f7fc844a8d8434c402c
+  data.tar.gz: 22470cf8c07ef38db0b2036853732ca527212c5abe03e2613021a497e1f320be534272ddafcb8b3c9c740005bf42a18b3edd22c63b7bcb026205865ee392ea4d

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - jruby-19mode

data/lib/logstash-input-kinesis_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/lib/logstash/inputs/kinesis/version.rb

@@ -1,7 +1,7 @@
 module Logstash
   module Input
     module Kinesis
-      VERSION = "1.3.0"
+      VERSION = "1.3.1"
     end
   end
 end

data/logstash-input-kinesis.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logstash/inputs/kinesis/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "logstash-input-kinesis"
+  spec.version       = Logstash::Input::Kinesis::VERSION
+  spec.authors       = ["Brian Palmer"]
+  spec.email         = ["brian@codekitchen.net"]
+  spec.summary       = %q{Logstash plugin for Kinesis input}
+  spec.description   = %q{This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program}
+  spec.homepage      = "https://github.com/codekitchen/logstash-input-kinesis"
+  spec.licenses      = ['Apache License (2.0)']
+
+  spec.files         = `git ls-files`.split($\)
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # Special flag to let us know this is actually a logstash plugin
+  spec.metadata      = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  spec.platform      = 'java'
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.2.0"
+  spec.add_development_dependency "logstash-core", ">= 1.5.1"
+  spec.add_development_dependency "logstash-codec-json"
+end

data/pom.xml

@@ -0,0 +1,27 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>codekitchen</groupId>
+  <artifactId>logstash-input-kinesis</artifactId>
+  <version>dev</version>
+  <dependencies>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>amazon-kinesis-client</artifactId>
+      <version>1.2.1</version>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

data/spec/inputs/kinesis/worker_spec.rb

@@ -0,0 +1,58 @@
+require "logstash/plugin"
+require "logstash/inputs/kinesis"
+require "logstash/codecs/json"
+require "json"
+
+RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
+  subject!(:worker) { LogStash::Inputs::Kinesis::Worker.new(codec, queue, decorator, checkpoint_interval) }
+  let(:codec) { LogStash::Codecs::JSON.new() }
+  let(:queue) { Queue.new }
+  let(:decorator) { proc { |x| x["decorated"] = true; x } }
+  let(:checkpoint_interval) { 120 }
+  let(:checkpointer) { double('checkpointer', checkpoint: nil) }
+  let(:shard_id) { "xyz" }
+
+  it "honors the initialize java interface method contract" do
+    expect { worker.initialize(shard_id) }.to_not raise_error
+  end
+
+  def record(hash = { "message" => "test" })
+    encoder = java.nio.charset::Charset.forName("UTF-8").newEncoder()
+    data = encoder.encode(java.nio.CharBuffer.wrap(JSON.generate(hash)))
+    double(getData: data)
+  end
+
+  let(:record1) { record(id: "record1", message: "test1") }
+  let(:record2) { record(id: "record2", message: "test2") }
+
+  context "initialized" do
+    before do
+      worker.initialize(shard_id)
+    end
+
+    describe "#processRecords" do
+      it "decodes and queues each record with decoration" do
+        worker.processRecords([record1, record2], checkpointer)
+        m1 = queue.pop
+        m2 = queue.pop
+        expect(m1).to be_kind_of(LogStash::Event)
+        expect(m2).to be_kind_of(LogStash::Event)
+        expect(m1['id']).to eq("record1")
+        expect(m1['message']).to eq("test1")
+        expect(m1['decorated']).to eq(true)
+      end
+
+      it "checkpoints on interval" do
+        expect(checkpointer).to receive(:checkpoint).once
+        worker.processRecords([], checkpointer)
+
+        # not this time
+        worker.processRecords([], checkpointer)
+
+        allow(Time).to receive(:now).and_return(Time.now + 125)
+        expect(checkpointer).to receive(:checkpoint).once
+        worker.processRecords([], checkpointer)
+      end
+    end
+  end
+end

data/spec/inputs/kinesis_spec.rb

@@ -0,0 +1,82 @@
+require "logstash/plugin"
+require "logstash/inputs/kinesis"
+require "logstash/codecs/json"
+
+RSpec.describe "inputs/kinesis" do
+  KCL = com.amazonaws.services.kinesis.clientlibrary.lib.worker
+
+  # It's very difficult to directly test the Java class, with all its private members,
+  # so we subclass it here.
+  class TestKCLWorker < KCL::Worker
+    field_reader :metricsFactory, :recordProcessorFactory
+    def run
+    end
+  end
+
+  let(:config) {{
+    "application_name" => "my-processor",
+    "kinesis_stream_name" => "run-specs",
+    "codec" => codec,
+    "metrics" => metrics,
+    "checkpoint_interval_seconds" => 120,
+    "region" => "ap-southeast-1",
+  }}
+
+  subject!(:kinesis) { LogStash::Inputs::Kinesis.new(config, kcl_class) }
+  let(:kcl_class) { TestKCLWorker }
+  let(:metrics) { nil }
+  let(:codec) { LogStash::Codecs::JSON.new() }
+
+  it "registers without error" do
+    input = LogStash::Plugin.lookup("input", "kinesis").new("kinesis_stream_name" => "specs", "codec" => codec)
+    expect { input.register }.to_not raise_error
+  end
+
+  it "configures the KCL" do
+    kinesis.register
+    expect(kinesis.kcl_config.applicationName).to eq("my-processor")
+    expect(kinesis.kcl_config.streamName).to eq("run-specs")
+    expect(kinesis.kcl_config.regionName).to eq("ap-southeast-1")
+    expect(kinesis.kcl_config.initialPositionInStream).to eq(KCL::InitialPositionInStream::TRIM_HORIZON)
+  end
+
+  context "#run" do
+    let(:queue) { Queue.new }
+
+    before do
+      kinesis.register
+    end
+
+    it "clones the codec for each worker" do
+      expect(codec).to receive(:clone).once
+      kinesis.run(queue)
+      worker = kinesis.kcl_worker.recordProcessorFactory.call()
+      expect(worker).to be_kind_of(LogStash::Inputs::Kinesis::Worker)
+    end
+
+    it "generates a valid worker via the factory proc" do
+      kinesis.run(queue)
+      worker = kinesis.kcl_worker.recordProcessorFactory.call()
+      expect(worker.codec).to be_kind_of(codec.class)
+      expect(worker.checkpoint_interval).to eq(120)
+      expect(worker.output_queue).to eq(queue)
+      expect(worker.decorator).to eq(kinesis.method(:decorate))
+      expect(worker.logger).to eq(kinesis.logger)
+    end
+
+    it "disables metric tracking by default" do
+      kinesis.run(queue)
+      expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::NullMetricsFactory)
+    end
+
+    context "cloudwatch" do
+      let(:metrics) { "cloudwatch" }
+      it "uses cloudwatch metrics if specified" do
+        kinesis.run(queue)
+        expect(kinesis.kcl_worker.metricsFactory).to be_kind_of(com.amazonaws.services.kinesis.metrics.impl::CWMetricsFactory)
+        # the process hangs otherwise
+        kinesis.kcl_worker.metricsFactory.shutdown
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.disable_monkey_patching!
+  config.warnings = false
+  if config.files_to_run.one?
+    config.default_formatter = 'doc'
+  end
+  config.order = :random
+  Kernel.srand config.seed
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: java
 authors:
 - Brian Palmer
@@ -87,13 +87,76 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- .rspec
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/logstash-input-kinesis_jars.rb
 - lib/logstash/inputs/kinesis.rb
 - lib/logstash/inputs/kinesis/version.rb
 - lib/logstash/inputs/kinesis/worker.rb
+- logstash-input-kinesis.gemspec
+- pom.xml
+- spec/inputs/kinesis/worker_spec.rb
+- spec/inputs/kinesis_spec.rb
+- spec/spec_helper.rb
+- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.2.1.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-autoscaling-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudformation-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudfront-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudhsm-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudsearch-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudtrail-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatchmetrics-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-codedeploy-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitoidentity-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cognitosync-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-config-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-datapipeline-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-directconnect-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ec2-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ecs-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticache-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticbeanstalk-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elasticloadbalancing-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-elastictranscoder-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-emr-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-glacier-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-iam-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-importexport-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-lambda-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-logs-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-opsworks-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-rds-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-redshift-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-route53-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-ses-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpledb-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-simpleworkflow-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sns-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sqs-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-storagegateway-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-support-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-swf-libraries-1.9.16.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.6.jar
+- vendor/jar-dependencies/runtime-jars/commons-logging-1.1.3.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.3.4.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-4.3.2.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.3.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.3.2.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.3.2.jar
+- vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
 homepage: https://github.com/codekitchen/logstash-input-kinesis
 licenses:
 - Apache License (2.0)
@@ -120,4 +183,7 @@ rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Logstash plugin for Kinesis input
-test_files: []
+test_files:
+- spec/inputs/kinesis/worker_spec.rb
+- spec/inputs/kinesis_spec.rb
+- spec/spec_helper.rb