logstash-output-kinesis 5.0.0-java → 5.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c560333a0ab8f30a9e65cbea047f1a2c5e27fabe
-  data.tar.gz: cc10598380a7c831621151aea4c1b5ad74cbaa2b
+  metadata.gz: 513e0b9ffc0a72b44051a083e5ea937bf2b2ae63
+  data.tar.gz: 903e4d155719c65f5ad986788895c779a427d1f8
 SHA512:
-  metadata.gz: c8e04c30bfc2d355b4534b108629aae2b6879ae58cc6eaafc5cddc4b54d9bd9e0135fcfe2889b1c6a93bbec49953cea1c86b84a9686abd0c5e75e89c1e1fc219
-  data.tar.gz: 14044649e49beb980b93b91ef3d6f270dba8caa5cfd90287a2546d4308a8e48a90893e5931088b9923ffd2defb06826ba09e8c2af8becea3755e30ce5b55fb5f
+  metadata.gz: 8dc90ea37fd160df8fdabab671f5017bd94ca1196ba52ddc5def9ecc7f6318904138626fe926a6244169a9ba57498db7b7aa22f838e65314d3903845fec445fc
+  data.tar.gz: 34ebcb7ee8a6a11f89eee1ae7c57671b8a539e9f865b51eb0a984c2b39effb7e3d24779d90df30ac442c5cd26d856c508d96aaa7f07381384f9a604d64644a67

data/.github_changelog_generator

@@ -0,0 +1 @@
+since-tag=v1.5.0

data/.travis.yml

@@ -1,6 +1,13 @@
 language: ruby
 jdk:
   - oraclejdk8
+before_install:
+  - mkdir -p $HOME/bin
+  - wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz
+  - tar xvfz git-lfs-linux-amd64-1.1.2.tar.gz
+  - mv git-lfs-1.1.2/git-lfs $HOME/bin/git-lfs
+  - export PATH=$PATH:$HOME/bin/
+  - git lfs pull
 deploy:
   provider: rubygems
   api_key:

data/CHANGELOG.md

@@ -0,0 +1,58 @@
+# Change Log
+
+## [5.1.0](https://github.com/samcday/logstash-output-kinesis/tree/5.1.0) (2017-09-01)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v5.0.0...5.1.0)
+
+**Closed issues:**
+
+- Is it possible send data via a proxy server to kinesis. [\#23](https://github.com/samcday/logstash-output-kinesis/issues/23)
+- logstash-output-kinesis does not work inside alpine linux [\#21](https://github.com/samcday/logstash-output-kinesis/issues/21)
+
+**Merged pull requests:**
+
+- Support sprintf-style stream names [\#22](https://github.com/samcday/logstash-output-kinesis/pull/22) ([sonya](https://github.com/sonya))
+- Plugin causes x-pack monitoring pipeline to fail  [\#20](https://github.com/samcday/logstash-output-kinesis/pull/20) ([jjensn](https://github.com/jjensn))
+
+## [v5.0.0](https://github.com/samcday/logstash-output-kinesis/tree/v5.0.0) (2017-06-10)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.6.0...v5.0.0)
+
+**Closed issues:**
+
+- Is it compatible with Logstash 5.x? [\#17](https://github.com/samcday/logstash-output-kinesis/issues/17)
+- Error Writing to Kinesis on EMR spinup [\#16](https://github.com/samcday/logstash-output-kinesis/issues/16)
+- Installing kinesis output plugin on a amazon linux box... [\#14](https://github.com/samcday/logstash-output-kinesis/issues/14)
+- Added to logstash-plugins? [\#13](https://github.com/samcday/logstash-output-kinesis/issues/13)
+- bundle error [\#12](https://github.com/samcday/logstash-output-kinesis/issues/12)
+- Support to produce UTF-8 JSON [\#11](https://github.com/samcday/logstash-output-kinesis/issues/11)
+- Kinesis is too busy? [\#10](https://github.com/samcday/logstash-output-kinesis/issues/10)
+- jar dependencies conflict with elasticsearch output plugin [\#9](https://github.com/samcday/logstash-output-kinesis/issues/9)
+- Invalid Signature Exception [\#8](https://github.com/samcday/logstash-output-kinesis/issues/8)
+- Logstash & Kinesis losing logs [\#7](https://github.com/samcday/logstash-output-kinesis/issues/7)
+- Not all records make it into Kinesis unless I pass the --debug flag to logstash [\#6](https://github.com/samcday/logstash-output-kinesis/issues/6)
+
+**Merged pull requests:**
+
+- Support logstash 5 [\#18](https://github.com/samcday/logstash-output-kinesis/pull/18) ([gregsterin](https://github.com/gregsterin))
+
+## [v1.6.0](https://github.com/samcday/logstash-output-kinesis/tree/v1.6.0) (2015-12-05)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.1.0...v1.6.0)
+
+## [v2.1.0](https://github.com/samcday/logstash-output-kinesis/tree/v2.1.0) (2015-12-05)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.0.1...v2.1.0)
+
+## [v2.0.1](https://github.com/samcday/logstash-output-kinesis/tree/v2.0.1) (2015-11-06)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.5.1...v2.0.1)
+
+## [v1.5.1](https://github.com/samcday/logstash-output-kinesis/tree/v1.5.1) (2015-11-06)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.0.0...v1.5.1)
+
+**Closed issues:**
+
+- Dropping lots of messages [\#5](https://github.com/samcday/logstash-output-kinesis/issues/5)
+
+## [v2.0.0](https://github.com/samcday/logstash-output-kinesis/tree/v2.0.0) (2015-10-31)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.5.0...v2.0.0)
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at me@samcday.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+## TL;DR
+
+Raising an issue is great, raising a PR is better, raising a PR with tests is *bestest*.
+
+## Developing
+
+You'll need [Git LFS](https://git-lfs.github.com/) to properly clone this repo.
+
+Ensure you have JRuby 9.1.x installed. [rvm](https://rvm.io/) is your friend :)
+
+```sh
+rvm use --install .
+gem install bundler && bundle install
+bundle exec rake
+```
+
+### Running tests
+
+```
+rake
+```
+
+### Updating KPL
+
+Change the dependency version in `build.gradle`, and then run `gradle copylibs`. Make sure to check in all the updated JARs!

data/README.md

@@ -28,6 +28,18 @@ This plugin accepts a wide range of configuration options, most of which come fr
 
 Please note that configuration options are snake_cased instead of camelCased. So, where [KinesisProducerConfiguration][kpldoc] offers a `setMetricsLevel` option, this plugin accepts a `metrics_level` option.
 
+### Dynamic stream name
+
+You can dictate the name of the stream to send a record to, based on data in the record itself.
+
+```nginx
+output {
+  kinesis {
+    stream_name => "%{myfield}-%{myotherfield}"
+  }
+}
+```
+
 ### Metrics
 
 The underlying KPL library defaults to sending CloudWatch metrics to give insight into what it's actually doing at runtime. It's highly recommended you ensure these metrics are flowing through, and use them to monitor the health of your log shipping.
@@ -151,6 +163,10 @@ output {
 
 ## Known Issues
 
+### Alpine Linux is not supported
+
+This Logstash plugin uses the KPL daemon under the covers. This daemon is linked against, and specifically requires, glibc. See awslabs/amazon-kinesis-producer#86.
+
 ### Noisy shutdown
 
 During shutdown of Logstash, you might get noisy warnings like this:
@@ -168,7 +184,7 @@ at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:61
 at java.lang.Thread.run(Thread.java:724)
 ```
 
-This is caused by [amazon-kinesis-producer#10](https://github.com/awslabs/amazon-kinesis-producer/issues/10)
+This is caused by awslabs/amazon-kinesis-producer#10.
 
 ### Noisy warnings about `Error during socket read`
 
@@ -178,27 +194,12 @@ While your Logstash instance is running, you may occasionally get a warning on s
 [2015-10-20 06:31:08.441640] [0x00007f36c9402700] [error] [io_service_socket.h:229] Error during socket read: End of file; 0 bytes read so far (kinesis.us-west-1.amazonaws.com:443)
 ```
 
-This is being tracked in [amazon-kinesis-producer#17](https://github.com/awslabs/amazon-kinesis-producer/issues/17). This log message seems to just be noise - your logs should still be delivering to Kinesis fine (but of course, you should independently verify this!).
-
-
-## Developing
-
-Ensure you have JRuby 9.1.x installed. [rvm](https://rvm.io/) is your friend :)
-
-```sh
-rvm use --install .
-gem install bundler && bundle install
-bundle exec rake
-```
-
-### Updating KPL
-
-Change the dependency version in `build.gradle`, and then run `gradle copylibs`. Make sure to check in all the updated JARs! Yes, we put them in the repo :(
+This is being tracked in awslabs/amazon-kinesis-producer#17. This log message seems to just be noise - your logs should still be delivering to Kinesis fine (but of course, you should independently verify this!).
 
 
 ## Contributions
 
-Are more than welcome. Raising an issue is great, raising a PR is better, raising a PR with tests is best.
+Are more than welcome. See [CONTRIBUTING.MD](CONTRIBUTING.MD)
 
 
 ## License

data/lib/logstash-output-kinesis/version.rb

@@ -1,3 +1,3 @@
 module LogstashOutputKinesis
-  VERSION = "5.0.0"
+  VERSION = "5.1.0"
 end

data/lib/logstash/outputs/kinesis.rb

@@ -173,7 +173,7 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def create_credentials_provider
     provider = AWSAuth.DefaultAWSCredentialsProviderChain.new()
     if @access_key and @secret_key
-      provider = BasicCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@access_key, @secret_key))
+      provider = BasicKinesisCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@access_key, @secret_key))
     end
     if @role_arn
       provider = create_sts_provider(provider, @role_arn)
@@ -184,7 +184,7 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def create_metrics_credentials_provider
     provider = AWSAuth.DefaultAWSCredentialsProviderChain.new()
     if @metrics_access_key and @metrics_secret_key
-      provider = BasicCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@metrics_access_key, @metrics_secret_key))
+      provider = BasicKinesisCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@metrics_access_key, @metrics_secret_key))
     end
     if @metrics_role_arn
       provider = create_sts_provider(provider, @metrics_role_arn)
@@ -195,7 +195,7 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   def send_record(event, payload)
     begin
       event_blob = ByteBuffer::wrap(payload.to_java_bytes)
-      @producer.addUserRecord(@stream_name, event.get("[@metadata][partition_key]"), event_blob)
+      @producer.addUserRecord(event.sprintf(@stream_name), event.get("[@metadata][partition_key]"), event_blob)
     rescue => e
       @logger.warn("Error writing event to Kinesis", :exception => e)
     end
@@ -209,7 +209,7 @@ class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
   end
 end
 
-class BasicCredentialsProvider
+class BasicKinesisCredentialsProvider
   java_implements 'com.amazonaws.auth.AWSCredentialsProvider'
 
   def initialize(credentials)

data/logstash-output-kinesis.gemspec

@@ -31,5 +31,4 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-plain", "< 6.0.0"
   s.add_runtime_dependency "logstash-codec-json", "< 6.0.0"
   s.add_development_dependency "logstash-devutils"
-  s.add_development_dependency "gem-release", "~>0.7.3"
 end

data/spec/outputs/kinesis_spec.rb

@@ -10,10 +10,30 @@ describe LogStash::Outputs::Kinesis do
     "region" => "ap-southeast-2",
     "metrics_level" => "none"
   }}
-  let(:sample_event) { LogStash::Event.new }
+  let(:sample_event) {
+    LogStash::Event.new({
+      "message" => "hello",
+      'stream_name' => 'my_stream',
+      "field1"  => "foo",
+      "field2"  => "bar"
+    })
+  }
 
   KPL = com.amazonaws.services.kinesis.producer
 
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "kinesis").new(config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate config with default values' do
+      output = LogStash::Outputs::Kinesis.new(config)
+      insist { output.randomized_partition_key } == false
+      insist { output.event_partition_keys } == []
+    end
+  end
+
   context "when receiving message" do
     it "sends record to Kinesis" do
       expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
@@ -23,5 +43,54 @@ describe LogStash::Outputs::Kinesis do
       output.receive(sample_event)
       output.close
     end
+
+    it "should support Event#sprintf placeholders in stream_name" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with("my_stream", anything, anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "stream_name" => "%{stream_name}",
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support blank partition keys" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, "-", anything)
+
+      output = LogStash::Outputs::Kinesis.new(config)
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support randomized partition keys" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, /[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+/, anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "randomized_partition_key" => true
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support fixed partition keys" do
+      # the partition key ends up being an empty string plus the first field
+      # we choose joined by a hyphen. this is a holdover from earlier versions
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, "-foo", anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "event_partition_keys" => ["[field1]", "[field2]"]
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
   end
+
 end

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kinesis
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.1.0
 platform: java
 authors:
 - Sam Day
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-10 00:00:00.000000000 Z
+date: 2017-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,9 +20,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
-  name: logstash-core
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -31,62 +31,50 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 6.0.0
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
-  name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
-  name: logstash-codec-json
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.3
-  name: gem-release
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.3
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis. This gem is not a stand-alone program
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
+  This gem is not a stand-alone program
 email: me@samcday.com.au
 executables: []
 extensions: []
@@ -95,11 +83,15 @@ files:
 - ".editorconfig"
 - ".gemrelease"
 - ".gitattributes"
+- ".github_changelog_generator"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
 - ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
 - Gemfile
 - LICENSE
 - README.md
@@ -136,7 +128,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -151,11 +143,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
 specification_version: 4
-summary: This output plugin sends records to Kinesis using the Kinesis Producer Library (KPL)
+summary: This output plugin sends records to Kinesis using the Kinesis Producer Library
+  (KPL)
 test_files:
 - spec/outputs/kinesis_spec.rb
 - spec/spec_helper.rb