logstash-input-kinesis 2.0.1-java → 2.0.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e58c88a1472fb5a313445353cd29777d014356a8
-  data.tar.gz: 6874ad08fd88c0f4e7d0924d024e5974b3cf5515
+  metadata.gz: 635b2940a469d0ec067ddadcf8cb4cc5bfc36560
+  data.tar.gz: 16bbc4801f5aa799348d7869070194a5513c4c03
 SHA512:
-  metadata.gz: 4e4bb72505af25b89a26bce1af2ce870007c65373138be07a188ae983cc0af818c2db7ed08d8dbc284b50b1d88424e4f272e96d97063894b73ecc22d2bc01cfe
-  data.tar.gz: 558f2ee9456e49b130e1ec33b0890c7a3485e46b2b2357c10862d011d0b9b4dcc15acfb937495996f54e8f6730d8aaf361c985e21292ce87ca20bfdd44ef38ed
+  metadata.gz: b940b70b465f724e59500df4a951688fb3fa37e17a01bfecb8d2ab8065c4360069c332e2e221afa271025e7b725f496a42cb4492439bf157117a6e91b60f0659
+  data.tar.gz: c8be8aa5fe5bfe81fd948775374266bc2e5456ed1c42006e9badb267c152ce6830e5d9966ea0b53ba6ec1c6b6b1bd86df38df21229a821c6fd9d01ca29156f19

data/README.md

@@ -10,7 +10,7 @@ This plugin requires Logstash >= 2.0, and can be installed by Logstash
 itself.
 
 ```sh
-$(LOGSTASH_DIR)/bin/plugin install logstash-input-kinesis
+bin/logstash-plugin install logstash-input-kinesis
 ```
 
 ## Usage
@@ -24,6 +24,10 @@ input {
 }
 ```
 
+### Using with CloudWatch Logs
+
+If you are looking to read a CloudWatch Logs subscription stream, you'll also want to install and configure the [CloudWatch Logs Codec](https://github.com/threadwaste/logstash-codec-cloudwatch_logs).
+
 ## Configuration
 
 This are the properties you can configure and what are the default values:

data/lib/logstash/inputs/kinesis.rb

@@ -3,10 +3,12 @@ require "logstash/inputs/base"
 require "logstash/errors"
 require "logstash/environment"
 require "logstash/namespace"
+require 'logstash-core-plugin-api/version'
 
 require 'logstash-input-kinesis_jars'
 require "logstash/inputs/kinesis/version"
 
+
 # Receive events through an AWS Kinesis stream.
 #
 # This input plugin uses the Java Kinesis Client Library underneath, so the
@@ -55,8 +57,13 @@ class LogStash::Inputs::Kinesis < LogStash::Inputs::Base
 
   def register
     # the INFO log level is extremely noisy in KCL
-    org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
-      logger.setLevel(java.util.logging::Level::WARNING)
+    if Gem::Version.new(LOGSTASH_CORE_PLUGIN_API) > Gem::Version.new('2.1.12')
+      org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
+        logger.setLevel(org.apache.log4j::Level::WARN)
+    else
+      org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").
+        logger.setLevel(java.util.logging::Level::WARNING)
+    end
 
     worker_id = java.util::UUID.randomUUID.to_s
     creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new

data/lib/logstash/inputs/kinesis/version.rb

@@ -2,7 +2,7 @@
 module Logstash
   module Input
     module Kinesis
-      VERSION = "2.0.1"
+      VERSION = "2.0.2"
     end
   end
 end

data/lib/logstash/inputs/kinesis/worker.rb

@@ -23,7 +23,7 @@ class LogStash::Inputs::Kinesis::Worker
   public :initialize
 
   def processRecords(records_input)
-    records_input.records.each { |record| process_record(record.getData) }
+    records_input.records.each { |record| process_record(record) }
     if Time.now >= @next_checkpoint
       checkpoint(records_input.checkpointer)
       @next_checkpoint = Time.now + @checkpoint_interval
@@ -45,12 +45,23 @@ class LogStash::Inputs::Kinesis::Worker
   end
 
   def process_record(record)
-    raw = String.from_java_bytes(record.array)
+    raw = String.from_java_bytes(record.getData.array)
+    metadata = build_metadata(record)
     @codec.decode(raw) do |event|
       @decorator.call(event)
+      event.set('@metadata', metadata)
       @output_queue << event
     end
   rescue => error
     @logger.error("Error processing record: #{error}")
   end
+
+  def build_metadata(record)
+    metadata = Hash.new
+    metadata['approximate_arrival_timestamp'] = record.getApproximateArrivalTimestamp.getTime
+    metadata['partition_key'] = record.getPartitionKey
+    metadata['sequence_number'] = record.getSequenceNumber
+    metadata
+  end
+
 end

data/logstash-input-kinesis.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.files         = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib', 'vendor/jar-dependencies/runtime-jars']
 
   # Special flag to let us know this is actually a logstash plugin
   spec.metadata      = { "logstash_plugin" => "true", "logstash_group" => "input" }
@@ -28,6 +28,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   spec.add_development_dependency 'logstash-devutils'
-  spec.add_development_dependency 'jar-dependencies', '0.3.2'
+  spec.add_development_dependency 'jar-dependencies', '~> 0.3.4'
   spec.add_development_dependency "logstash-codec-json"
 end

data/spec/inputs/kinesis/worker_spec.rb

@@ -19,18 +19,38 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
     expect { worker.initialize(init_input) }.to_not raise_error
   end
 
-  def record(hash = { "message" => "test" })
+  def record(hash = { "message" => "test" }, arrival_timestamp, partition_key, sequence_number)
     encoder = java.nio.charset::Charset.forName("UTF-8").newEncoder()
     data = encoder.encode(java.nio.CharBuffer.wrap(JSON.generate(hash)))
-    double(getData: data)
+    double(
+      getData: data,
+      getApproximateArrivalTimestamp: java.util.Date.new(arrival_timestamp.to_f * 1000),
+      getPartitionKey: partition_key,
+      getSequenceNumber: sequence_number
+    )
   end
 
   let(:process_input) {
     KCL_TYPES::ProcessRecordsInput.new()
         .withRecords(java.util.Arrays.asList([
-            record(id: "record1", message: "test1"),
-            record(id: "record2", message: "test2")
-          ].to_java)
+          record(
+            {
+              id: "record1",
+              message: "test1"
+            },
+            '1.441215410867E9',
+            'partitionKey1',
+            '21269319989652663814458848515492872191'
+          ),
+          record(
+            {
+              id: "record2",
+              message: "test2"
+            },
+            '1.441215410868E9',
+            'partitionKey2',
+            '21269319989652663814458848515492872192'
+          )].to_java)
         )
         .withCheckpointer(checkpointer)
   }
@@ -55,6 +75,9 @@ RSpec.describe "LogStash::Inputs::Kinesis::Worker" do
         expect(m2).to be_kind_of(LogStash::Event)
         expect(m1.get('id')).to eq("record1")
         expect(m1.get('message')).to eq("test1")
+        expect(m1.get('@metadata')['approximate_arrival_timestamp']).to eq(1441215410867)
+        expect(m1.get('@metadata')['partition_key']).to eq('partitionKey1')
+        expect(m1.get('@metadata')['sequence_number']).to eq('21269319989652663814458848515492872191')
         expect(m1.get('decorated')).to eq(true)
       end
 

data/vendor/jar-dependencies/runtime-jars/logstash-input-kinesis_jars.rb

@@ -0,0 +1,46 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'com/amazonaws/aws-java-sdk-kms/1.11.14/aws-java-sdk-kms-1.11.14.jar'
+  require 'com/amazonaws/amazon-kinesis-client/1.7.0/amazon-kinesis-client-1.7.0.jar'
+  require 'com/amazonaws/aws-java-sdk-core/1.11.16/aws-java-sdk-core-1.11.16.jar'
+  require 'commons-codec/commons-codec/1.9/commons-codec-1.9.jar'
+  require 'com/fasterxml/jackson/core/jackson-databind/2.6.6/jackson-databind-2.6.6.jar'
+  require 'com/google/guava/guava/18.0/guava-18.0.jar'
+  require 'commons-lang/commons-lang/2.6/commons-lang-2.6.jar'
+  require 'commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar'
+  require 'joda-time/joda-time/2.8.1/joda-time-2.8.1.jar'
+  require 'com/amazonaws/aws-java-sdk-s3/1.11.14/aws-java-sdk-s3-1.11.14.jar'
+  require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.6/jackson-dataformat-cbor-2.6.6.jar'
+  require 'com/fasterxml/jackson/core/jackson-core/2.6.6/jackson-core-2.6.6.jar'
+  require 'com/amazonaws/aws-java-sdk-dynamodb/1.11.14/aws-java-sdk-dynamodb-1.11.14.jar'
+  require 'com/amazonaws/aws-java-sdk-cloudwatch/1.11.14/aws-java-sdk-cloudwatch-1.11.14.jar'
+  require 'com/amazonaws/aws-java-sdk-kinesis/1.11.14/aws-java-sdk-kinesis-1.11.14.jar'
+  require 'org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar'
+  require 'org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar'
+  require 'com/fasterxml/jackson/core/jackson-annotations/2.6.0/jackson-annotations-2.6.0.jar'
+  require 'com/google/protobuf/protobuf-java/2.6.1/protobuf-java-2.6.1.jar'
+end
+
+if defined? Jars
+  require_jar( 'com.amazonaws', 'aws-java-sdk-kms', '1.11.14' )
+  require_jar( 'com.amazonaws', 'amazon-kinesis-client', '1.7.0' )
+  require_jar( 'com.amazonaws', 'aws-java-sdk-core', '1.11.16' )
+  require_jar( 'commons-codec', 'commons-codec', '1.9' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.6.6' )
+  require_jar( 'com.google.guava', 'guava', '18.0' )
+  require_jar( 'commons-lang', 'commons-lang', '2.6' )
+  require_jar( 'commons-logging', 'commons-logging', '1.1.3' )
+  require_jar( 'joda-time', 'joda-time', '2.8.1' )
+  require_jar( 'com.amazonaws', 'aws-java-sdk-s3', '1.11.14' )
+  require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.6.6' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.6.6' )
+  require_jar( 'com.amazonaws', 'aws-java-sdk-dynamodb', '1.11.14' )
+  require_jar( 'com.amazonaws', 'aws-java-sdk-cloudwatch', '1.11.14' )
+  require_jar( 'com.amazonaws', 'aws-java-sdk-kinesis', '1.11.14' )
+  require_jar( 'org.apache.httpcomponents', 'httpclient', '4.5.2' )
+  require_jar( 'org.apache.httpcomponents', 'httpcore', '4.4.4' )
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.6.0' )
+  require_jar( 'com.google.protobuf', 'protobuf-java', '2.6.1' )
+end

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-kinesis
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: java
 authors:
 - Brian Palmer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-26 00:00:00.000000000 Z
+date: 2016-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -41,27 +41,27 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.4
   name: jar-dependencies
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.4
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-json
@@ -69,7 +69,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -79,36 +79,54 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- CONTRIBUTORS
-- Gemfile
-- README.md
-- lib/logstash-input-kinesis_jars.rb
 - lib/logstash/inputs/kinesis.rb
 - lib/logstash/inputs/kinesis/version.rb
 - lib/logstash/inputs/kinesis/worker.rb
-- logstash-input-kinesis.gemspec
-- spec/inputs/kinesis/worker_spec.rb
-- spec/inputs/kinesis_spec.rb
 - spec/spec_helper.rb
-- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.7.0.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.11.14.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.11.16.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.11.14.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.11.14.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.11.14.jar
-- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.11.14.jar
+- spec/inputs/kinesis_spec.rb
+- spec/inputs/kinesis/worker_spec.rb
+- vendor/jar-dependencies/runtime-jars/amazon-kinesis-client-1.6.1.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-cloudwatch-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-dynamodb-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kinesis-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-kms-1.10.20.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-s3-1.10.20.jar
 - vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
 - vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar
-- vendor/jar-dependencies/runtime-jars/commons-logging-1.1.3.jar
+- vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
 - vendor/jar-dependencies/runtime-jars/guava-18.0.jar
-- vendor/jar-dependencies/runtime-jars/httpclient-4.5.2.jar
-- vendor/jar-dependencies/runtime-jars/httpcore-4.4.4.jar
-- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.6.0.jar
-- vendor/jar-dependencies/runtime-jars/jackson-core-2.6.6.jar
-- vendor/jar-dependencies/runtime-jars/jackson-databind-2.6.6.jar
-- vendor/jar-dependencies/runtime-jars/jackson-dataformat-cbor-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.5.1.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-4.4.3.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.5.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.5.3.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.5.3.jar
 - vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
+- vendor/jar-dependencies/runtime-jars/logstash-input-kinesis_jars.rb
 - vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/amazon-kinesis-client/1.7.0/amazon-kinesis-client-1.7.0.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-cloudwatch/1.11.14/aws-java-sdk-cloudwatch-1.11.14.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-core/1.11.16/aws-java-sdk-core-1.11.16.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-dynamodb/1.11.14/aws-java-sdk-dynamodb-1.11.14.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-kinesis/1.11.14/aws-java-sdk-kinesis-1.11.14.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-kms/1.11.14/aws-java-sdk-kms-1.11.14.jar
+- vendor/jar-dependencies/runtime-jars/com/amazonaws/aws-java-sdk-s3/1.11.14/aws-java-sdk-s3-1.11.14.jar
+- vendor/jar-dependencies/runtime-jars/com/fasterxml/jackson/core/jackson-annotations/2.6.0/jackson-annotations-2.6.0.jar
+- vendor/jar-dependencies/runtime-jars/com/fasterxml/jackson/core/jackson-core/2.6.6/jackson-core-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/com/fasterxml/jackson/core/jackson-databind/2.6.6/jackson-databind-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.6.6/jackson-dataformat-cbor-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/com/google/guava/guava/18.0/guava-18.0.jar
+- vendor/jar-dependencies/runtime-jars/com/google/protobuf/protobuf-java/2.6.1/protobuf-java-2.6.1.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec/commons-codec/1.9/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-lang/commons-lang/2.6/commons-lang-2.6.jar
+- vendor/jar-dependencies/runtime-jars/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar
+- vendor/jar-dependencies/runtime-jars/joda-time/joda-time/2.8.1/joda-time-2.8.1.jar
+- vendor/jar-dependencies/runtime-jars/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar
+- vendor/jar-dependencies/runtime-jars/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar
+- logstash-input-kinesis.gemspec
+- README.md
+- CONTRIBUTORS
+- Gemfile
 homepage: https://github.com/logstash-plugins/logstash-input-kinesis
 licenses:
 - Apache License (2.0)
@@ -119,25 +137,26 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
+- vendor/jar-dependencies/runtime-jars
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
 - jar 'com.amazonaws:amazon-kinesis-client', '1.7.0'
 - jar 'com.amazonaws:aws-java-sdk-core', '1.11.16'
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Logstash plugin for Kinesis input
 test_files:
-- spec/inputs/kinesis/worker_spec.rb
-- spec/inputs/kinesis_spec.rb
 - spec/spec_helper.rb
+- spec/inputs/kinesis_spec.rb
+- spec/inputs/kinesis/worker_spec.rb

data/lib/logstash-input-kinesis_jars.rb

@@ -1,5 +0,0 @@
-# encoding: utf-8
-require 'logstash/environment'
-
-root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")