logstash-output-kinesis-jdk11 5.1.3-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b96b2a50ecc741fc1aa6bbdb845dea9ed009d7909f4997f2f63bcd9f0c26a28
+  data.tar.gz: fecef040769c7b6a9a044f532f7d66b210ff703c27df75312f5cbf5373cff13c
+SHA512:
+  metadata.gz: 88062243a4f64d87def45db846cf4988fda458b2596a0e3960a01f7267aae44b7fbb9fb9988e25e3b666b9153ea0b06abdd8ca24d3a10988957b4f9665071c7c
+  data.tar.gz: d175865467feca1014fa1fb5c6e120f367ceab5c7a02ca4c04b1debad311cbb6a9f422db00488262adc7939af414d13f6135ab49367982c1f3067c6bcff5fc45

data/.editorconfig

@@ -0,0 +1,11 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{rb,gemspec}]
+indent_style = space
+indent_size = 2

data/.gemrelease

@@ -0,0 +1,3 @@
+bump:
+  tag: true
+  release: false

data/.gitattributes

@@ -0,0 +1 @@
+vendor/jar-dependencies/runtime-jars/*.jar filter=lfs diff=lfs merge=lfs -text

data/.github_changelog_generator

@@ -0,0 +1 @@
+since-tag=v1.5.0

data/.gitignore

@@ -0,0 +1,7 @@
+/.gradle/
+/*.gem
+.Gemfile.lock
+.Gemfile.bak
+/.bundle
+.DS_Store
+/Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+logstash-output-kinesis

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.3.4.0

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+jdk:
+  - oraclejdk8
+before_install:
+  - mkdir -p $HOME/bin
+  - wget https://github.com/github/git-lfs/releases/download/v1.1.2/git-lfs-linux-amd64-1.1.2.tar.gz
+  - tar xvfz git-lfs-linux-amd64-1.1.2.tar.gz
+  - mv git-lfs-1.1.2/git-lfs $HOME/bin/git-lfs
+  - export PATH=$PATH:$HOME/bin/
+  - git lfs pull
+deploy:
+  provider: rubygems
+  api_key:
+    secure: KLegBk+GpcGl4JJvWO2lre8xG/43RUjPlWVVCmvz0hmN33+d78WMRErIgGta9FgXw6KE7VPymXiFrS8EAdY5IHe1Oxt/PuvC1/FiP/oU7IPp1UcmqOxPdiOJtqFy7DqjfNc/nvmvUYf1JKKNMBAjSFgzzccQFB418wUPb1dokkl/QgS2azP3aF9PfKWiZd4iWUVxY30TB2W1+qvjT6ix6idU7aQjTwzFIvEs6WYZuIdl0fB2oF5no8oyw6NVfaxA89Jv2d8CGVlIOp6Q/ZXzNCviM/PYkKVuCmObgG19q4v8L2M0y+2GcLEORbwmwFSLBj6rjLg6WpZXCx+QppA7AeuQZ0rvZ+Jb4Eaqu+dQTd57We1dbcGWgonlisuqkJv0cv7Qy+hupJ5mil/ofTUoHapwhTpsdVch/XsV8rV3wu012j1eeTgp4EjbBcgHYnmc6XJGHLigVAEv//OQEl7eUpM7zZbmpaYmtjJsFveVP/WS8/rAwjytSD58mcVyNzHTtjVgr58ejUc61TvZFErkPhfUjtj+xuRyJKQXmS5Y70itRDBtu3VFNoimXg2Onz3bWmUQSQ7kL3/ctvXUoT8Rmza712bF1ZRblbKLi+sPFXo2++v4wxxOH89aqjdYcnjP+tBz3KCN9jVGJ9c8nUI08Lv29dJreQJl3N1Cjnce2cA=
+  gem: logstash-output-kinesis
+  on:
+    tags: true
+    repo: samcday/logstash-output-kinesis

data/CHANGELOG.md

@@ -0,0 +1,58 @@
+# Change Log
+
+## [5.1.0](https://github.com/samcday/logstash-output-kinesis/tree/5.1.0) (2017-09-01)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v5.0.0...5.1.0)
+
+**Closed issues:**
+
+- Is it possible send data via a proxy server to kinesis. [\#23](https://github.com/samcday/logstash-output-kinesis/issues/23)
+- logstash-output-kinesis does not work inside alpine linux [\#21](https://github.com/samcday/logstash-output-kinesis/issues/21)
+
+**Merged pull requests:**
+
+- Support sprintf-style stream names [\#22](https://github.com/samcday/logstash-output-kinesis/pull/22) ([sonya](https://github.com/sonya))
+- Plugin causes x-pack monitoring pipeline to fail  [\#20](https://github.com/samcday/logstash-output-kinesis/pull/20) ([jjensn](https://github.com/jjensn))
+
+## [v5.0.0](https://github.com/samcday/logstash-output-kinesis/tree/v5.0.0) (2017-06-10)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.6.0...v5.0.0)
+
+**Closed issues:**
+
+- Is it compatible with Logstash 5.x? [\#17](https://github.com/samcday/logstash-output-kinesis/issues/17)
+- Error Writing to Kinesis on EMR spinup [\#16](https://github.com/samcday/logstash-output-kinesis/issues/16)
+- Installing kinesis output plugin on a amazon linux box... [\#14](https://github.com/samcday/logstash-output-kinesis/issues/14)
+- Added to logstash-plugins? [\#13](https://github.com/samcday/logstash-output-kinesis/issues/13)
+- bundle error [\#12](https://github.com/samcday/logstash-output-kinesis/issues/12)
+- Support to produce UTF-8 JSON [\#11](https://github.com/samcday/logstash-output-kinesis/issues/11)
+- Kinesis is too busy? [\#10](https://github.com/samcday/logstash-output-kinesis/issues/10)
+- jar dependencies conflict with elasticsearch output plugin [\#9](https://github.com/samcday/logstash-output-kinesis/issues/9)
+- Invalid Signature Exception [\#8](https://github.com/samcday/logstash-output-kinesis/issues/8)
+- Logstash & Kinesis losing logs [\#7](https://github.com/samcday/logstash-output-kinesis/issues/7)
+- Not all records make it into Kinesis unless I pass the --debug flag to logstash [\#6](https://github.com/samcday/logstash-output-kinesis/issues/6)
+
+**Merged pull requests:**
+
+- Support logstash 5 [\#18](https://github.com/samcday/logstash-output-kinesis/pull/18) ([gregsterin](https://github.com/gregsterin))
+
+## [v1.6.0](https://github.com/samcday/logstash-output-kinesis/tree/v1.6.0) (2015-12-05)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.1.0...v1.6.0)
+
+## [v2.1.0](https://github.com/samcday/logstash-output-kinesis/tree/v2.1.0) (2015-12-05)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.0.1...v2.1.0)
+
+## [v2.0.1](https://github.com/samcday/logstash-output-kinesis/tree/v2.0.1) (2015-11-06)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.5.1...v2.0.1)
+
+## [v1.5.1](https://github.com/samcday/logstash-output-kinesis/tree/v1.5.1) (2015-11-06)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v2.0.0...v1.5.1)
+
+**Closed issues:**
+
+- Dropping lots of messages [\#5](https://github.com/samcday/logstash-output-kinesis/issues/5)
+
+## [v2.0.0](https://github.com/samcday/logstash-output-kinesis/tree/v2.0.0) (2015-10-31)
+[Full Changelog](https://github.com/samcday/logstash-output-kinesis/compare/v1.5.0...v2.0.0)
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at me@samcday.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/CONTRIBUTING.md

@@ -0,0 +1,36 @@
+## TL;DR
+
+Raising an issue is great, raising a PR is better, raising a PR with tests is *bestest*.
+
+## Developing
+
+You'll need [Git LFS](https://git-lfs.github.com/) to properly clone this repo.
+
+Ensure you have JRuby 9.1.x installed. [rvm](https://rvm.io/) is your friend :)
+
+```sh
+rvm use --install .
+gem install bundler && bundle install
+bundle exec rake
+```
+
+### Running tests
+
+```
+rake
+```
+
+### Building gem
+
+```
+gem build logstash-output-kinesis
+```
+
+### Testing locally built gem
+```
+bin/logstash-plugin install --local /path/to/logstash-output-kinesis-5.1.1-java.gem
+```
+
+### Updating KPL
+
+Change the dependency version in `build.gradle`, and then run `gradle copylibs`. Make sure to check in all the updated JARs!

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Sam Day <http://www.samcday.com.au>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,230 @@
+# Kinesis Output Plugin
+
+[![Build Status][badge-travis]][travis]
+[![Gem info][badge-gem]][rubygems]
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash). It will send log records to a [Kinesis stream](https://aws.amazon.com/kinesis/), using the [Kinesis Producer Library (KPL)](https://docs.aws.amazon.com/kinesis/latest/dev/developing-producers-with-kpl.html).
+
+**This version is intended for use with Logstash 5.x.** For plugin versions compatible with older versions of Logstash:
+
+ * [Logstash 1.5.x](https://github.com/samcday/logstash-output-kinesis/tree/1.5)
+ * [Logstash 2.x](https://github.com/samcday/logstash-output-kinesis/tree/2.x)
+
+
+## Configuration
+
+Minimum required configuration to get this plugin chugging along:
+
+```nginx
+output {
+  kinesis {
+    stream_name => "logs-stream"
+    region => "ap-southeast-2"
+  }
+}
+```
+
+This plugin accepts a wide range of configuration options, most of which come from the underlying KPL library itself. [View the full list of KPL configuration options here.][kpldoc]
+
+Please note that configuration options are snake_cased instead of camelCased. So, where [KinesisProducerConfiguration][kpldoc] offers a `setMetricsLevel` option, this plugin accepts a `metrics_level` option.
+
+### Dynamic stream name
+
+You can dictate the name of the stream to send a record to, based on data in the record itself.
+
+```nginx
+output {
+  kinesis {
+    stream_name => "%{myfield}-%{myotherfield}"
+  }
+}
+```
+
+### Metrics
+
+The underlying KPL library defaults to sending CloudWatch metrics to give insight into what it's actually doing at runtime. It's highly recommended you ensure these metrics are flowing through, and use them to monitor the health of your log shipping.
+
+If for some reason you want to switch them off, you can easily do so:
+
+```nginx
+output {
+  kinesis {
+    # ...
+
+    metrics_level => "none"
+  }
+}
+```
+
+If you choose to keep metrics enabled, ensure the AWS credentials you provide to this plugin are able to write to Kinesis *and* write to CloudWatch.
+
+### Authentication
+
+By default, this plugin will use the AWS SDK [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) to obtain credentials for communication with the Kinesis stream (and CloudWatch, if metrics are enabled). The following places will be checked for credentials:
+
+ * `AWS_ACCESS_KEY_ID` / `AWS_SECRET_KEY` environment variables available to the Logstash prociess
+ * `~/.aws/credentials` credentials file
+ * Instance profile (if Logstash is running in an EC2 instance)
+
+If you want to provide credentials directly in the config file, you can do so:
+
+```nginx
+output {
+  kinesis {
+    # ...
+
+    access_key => "AKIAIDFAKECREDENTIAL"
+    secret_key => "KX0ofakeLcredentialsGrightJherepOlolPkQk"
+
+    # You can provide specific credentials for CloudWatch metrics:
+    metrics_access_key => "AKIAIDFAKECREDENTIAL"
+    metrics_secret_key => "KX0ofakeLcredentialsGrightJherepOlolPkQk"
+  }
+}
+```
+
+If `access_key` and `secret_key` are provided, they will be used for communicating with Kinesis *and* CloudWatch. If `metrics_access_key` and `metrics_secret_key` are provided, they will be used for communication with CloudWatch. If only the metrics credentials were provided, Kinesis would use the default credentials provider (explained above) and CloudWatch would use the specific credentials. Confused? Good!
+
+#### Using STS
+
+You can also configure this plugin to use [AWS STS](https://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html) to "assume" a role that has access to Kinesis and CloudWatch. If you use this in combination with EC2 instance profiles (which the defaults credentials provider explained above uses) then you can actually configure your Logstash to write to Kinesis and CloudWatch without any hardcoded credentials.
+
+```nginx
+output {
+  kinesis {
+    # ...
+
+    role_arn => "arn:aws:iam::123456789:role/my-kinesis-producer-role"
+
+    # You can also provide a specific role to assume for CloudWatch metrics:
+    metrics_role_arn => "arn:aws:iam::123456789:role/my-metrics-role"
+  }
+}
+```
+
+You can combine `role_arn` / `metrics_role_arn` with the explicit AWS credentials config explained earlier, too.
+
+All this stuff can be mixed too - if you wanted to use hardcoded credentials for Kinesis, but then assume a role via STS for accessing CloudWatch, you can do that. Vice versa would work too - assume a role for accessing Kinesis and then providing hardcoded credentials for CloudWatch. Make things as arbitrarily complicated for yourself as you like ;)
+
+### Building a partition key
+
+Kinesis demands a [partition key](https://docs.aws.amazon.com/kinesis/latest/dev/key-concepts.html#partition-key) be provided for each record. By default, this plugin will provide a very boring partition key of `-`. However, you can configure it to compute a partition key from fields in your log events.
+
+```nginx
+output {
+  kinesis {
+    # ...
+    event_partition_keys => ["[field1]", "[field2]"]
+  }
+}
+```
+
+#### Randomised partition keys
+
+If you don't care about the ordering of your logs in the Kinesis stream, you might want to use a random partition key. This way, your log stream will be more or less uniformly spread across all available shards in the Kinesis stream.
+
+```nginx
+output {
+  kinesis {
+    randomized_partition_key => true
+  }
+}
+```
+
+### Record Aggregation
+
+The [Amazon KPL library can aggregate](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-concepts.html#d0e3423) your records when writing to the Kinesis stream. **This behaviour is configured to be enabled by default.**
+
+If you are using an older version of the Amazon KCL library to consume your records, or not using KCL at all, your consumer application(s) will probably not behave correctly. See [the matrix on this page](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-integration.html) for more info, and read [more about de-aggregating records here](https://docs.aws.amazon.com/kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html).
+
+If you wish to simply disable record aggregation, that's easy:
+
+```nginx
+output {
+  kinesis {
+    aggregation_enabled => false
+  }
+}
+```
+
+### Backpressure
+
+This plugin will enforce backpressure if the records passing through Logstash's pipeline are not making it up to Kinesis fast enough. When this happens, Logstash will stop accepting records for input or filtering, and a warning will be emitted in the Logstash logs.
+
+By default, the threshold for blocking is 1000 pending records. If you want to throw more memory / CPU cycles at buffering lots of stuff before it makes it to Kinesis, you can control the high-watermark:
+
+```nginx
+output {
+  kinesis {
+    max_pending_records => 10000
+  }
+}
+```
+
+### Logging configuration
+
+The underlying KPL uses SLF4J for logging and binding for Log4j (used by Logstash) is included in the plugin package. Thus the logging levels can be controlled with the `log4j2.properties` file provided by Logstash.
+
+As the KPL might be too noisy with `INFO` level, you might want to dial it down by following configuration in `log4j2.properties`:
+
+```
+...
+logger.kinesis.name = com.amazonaws.services.kinesis
+logger.kinesis.level = WARN
+logger.kinesis.additivity = false
+logger.kinesis.appenderRef.console.ref = console
+...
+
+``` 
+
+## Known Issues
+
+### Alpine Linux is not supported
+
+This Logstash plugin uses the KPL daemon under the covers. This daemon is linked against, and specifically requires, glibc. See awslabs/amazon-kinesis-producer#86.
+
+### Noisy shutdown
+
+During shutdown of Logstash, you might get noisy warnings like this:
+
+```
+[pool-1-thread-6] WARN com.amazonaws.services.kinesis.producer.Daemon - Exception during updateCredentials
+java.lang.InterruptedException: sleep interrupted
+at java.lang.Thread.sleep(Native Method)
+at com.amazonaws.services.kinesis.producer.Daemon$5.run(Daemon.java:316)
+at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
+at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
+at java.util.concurrent.FutureTask.run(FutureTask.java:166)
+at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
+at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
+at java.lang.Thread.run(Thread.java:724)
+```
+
+This is caused by awslabs/amazon-kinesis-producer#10.
+
+### Noisy warnings about `Error during socket read`
+
+While your Logstash instance is running, you may occasionally get a warning on stderr that looks like this:
+
+```
+[2015-10-20 06:31:08.441640] [0x00007f36c9402700] [error] [io_service_socket.h:229] Error during socket read: End of file; 0 bytes read so far (kinesis.us-west-1.amazonaws.com:443)
+```
+
+This is being tracked in awslabs/amazon-kinesis-producer#17. This log message seems to just be noise - your logs should still be delivering to Kinesis fine (but of course, you should independently verify this!).
+
+
+## Contributions
+
+Are more than welcome. See [CONTRIBUTING.md](CONTRIBUTING.md)
+
+
+## License
+
+[Apache License 2.0](LICENSE)
+
+[travis]: https://travis-ci.org/samcday/logstash-output-kinesis
+[rubygems]: https://rubygems.org/gems/logstash-output-kinesis
+[kpldoc]: https://github.com/awslabs/amazon-kinesis-producer/blob/v0.12.5/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38
+
+[badge-travis]: https://img.shields.io/travis/samcday/logstash-output-kinesis.svg?style=flat-square
+[badge-gem]: https://img.shields.io/gem/v/logstash-output-kinesis.svg?style=flat-square

data/Rakefile

@@ -0,0 +1,8 @@
+require "logstash/devutils/rake"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = Dir.glob('spec/**/*_spec.rb')
+end
+
+task :default => :spec

data/build.gradle

@@ -0,0 +1,24 @@
+apply plugin: 'java'
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+configurations {
+  customConfig.extendsFrom implementation
+}
+
+dependencies {
+    implementation 'com.amazonaws:amazon-kinesis-producer:0.14.12'
+    implementation 'com.amazonaws:aws-java-sdk-sts:1.12.200'
+    implementation ('org.apache.logging.log4j:log4j-slf4j-impl:2.6.2' ) {
+        exclude group: 'org.apache.logging.log4j', module: 'log4j-api'
+        exclude group: 'org.slf4j', module: 'slf4j-api'
+    }
+}
+
+task copyLibs(type: Copy) {
+    from configurations.customConfig
+    into 'vendor/jar-dependencies/runtime-jars'
+}

data/lib/logstash/outputs/kinesis.rb

@@ -0,0 +1,228 @@
+# encoding: utf-8
+
+require "java"
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "securerandom"
+require "logstash-output-kinesis_jars"
+
+# Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
+# Most of the configuration options in this plugin are simply passed on to
+# link:https://github.com/awslabs/amazon-kinesis-producer/blob/v0.12.5/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38[KinesisProducerConfiguration]
+class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
+  config_name "kinesis"
+
+  default :codec, 'json'
+
+  # The name of the stream to send data to.
+  config :stream_name, :validate => :string, :required => true
+  # A list of event data keys to use when constructing a partition key
+  config :event_partition_keys, :validate => :array, :default => []
+  # If true, a random partition key will be assigned to each log record
+  config :randomized_partition_key, :validate => :boolean, :default => false
+  # If the number of records pending being written to Kinesis exceeds this number, then block
+  # Logstash processing until they're all written.
+  config :max_pending_records, :validate => :number, :default => 1000
+
+  # An AWS access key to use for authentication to Kinesis and CloudWatch
+  config :access_key, :validate => :string
+  # An AWS secret key to use for authentication to Kinesis and CloudWatch
+  config :secret_key, :validate => :string
+  # If provided, STS will be used to assume this role and use it to authenticate to Kinesis and CloudWatch
+  config :role_arn, :validate => :string
+
+  # If provided, use this AWS access key for authentication to CloudWatch
+  config :metrics_access_key, :validate => :string
+  # If provided, use this AWS secret key for authentication to CloudWatch
+  config :metrics_secret_key, :validate => :string
+  # If provided, STS will be used to assume this role and use it to authenticate to CloudWatch
+  config :metrics_role_arn, :validate => :string
+
+  config :sts_proxy_host, :validate => :string
+  config :sts_proxy_port, :validate => :number
+
+  config :aggregation_enabled, :validate => :boolean, :default => true
+  config :aggregation_max_count, :validate => :number, :default => 4294967295
+  config :aggregation_max_size, :validate => :number, :default => 51200
+  config :cloudwatch_endpoint, :validate => :string, :default => nil
+  config :cloudwatch_port, :validate => :number, :default => 443
+  config :collection_max_count, :validate => :number, :default => 500
+  config :collection_max_size, :validate => :number, :default => 5242880
+  config :connect_timeout, :validate => :number, :default => 6000
+  config :credentials_refresh_delay, :validate => :number, :default => 5000
+  config :enable_core_dumps, :validate => :boolean, :default => false
+  config :fail_if_throttled, :validate => :boolean, :default => false
+  config :kinesis_endpoint, :validate => :string, :default => nil
+  config :kinesis_port, :validate => :number, :default => 443
+  config :log_level, :validate => ["info", "warning", "error"], :default => "info"
+  config :max_connections, :validate => :number, :default => 4
+  config :metrics_granularity, :validate => ["global", "stream", "shard"], :default => "shard"
+  config :metrics_level, :validate => ["none", "summary", "detailed"], :default => "detailed"
+  config :metrics_namespace, :validate => :string, :default => "KinesisProducerLibrary"
+  config :metrics_upload_delay, :validate => :number, :default => 60000
+  config :min_connections, :validate => :number, :default => 1
+  config :native_executable, :validate => :string, :default => nil
+  config :rate_limit, :validate => :number, :default => 150
+  config :record_max_buffered_time, :validate => :number, :default => 100
+  config :record_ttl, :validate => :number, :default => 30000
+  config :region, :validate => :string, :required => true
+  config :request_timeout, :validate => :number, :default => 6000
+  config :temp_directory, :validate => :string, :default => nil
+  config :verify_certificate, :validate => :boolean, :default => true
+
+  KPL = com.amazonaws.services.kinesis.producer
+  AWSAuth = com.amazonaws.auth
+  ByteBuffer = java.nio.ByteBuffer
+
+  public
+  def register
+    @metrics_access_key ||= @access_key
+    @metrics_secret_key ||= @secret_key
+
+    @producer = KPL.KinesisProducer::new(create_kpl_config)
+    @codec.on_event(&method(:send_record))
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    if @randomized_partition_key
+      event.set("[@metadata][partition_key]", SecureRandom.uuid)
+    else
+      # Haha - gawd. If I don't put an empty string in the array, then calling .join()
+      # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
+      partition_key_parts = [""]
+
+      @event_partition_keys.each do |partition_key_name|
+        if not event.get(partition_key_name).nil? and event.get(partition_key_name).length > 0
+          partition_key_parts << event.get(partition_key_name).to_s
+          break
+        end
+      end
+
+      event.set("[@metadata][partition_key]", (partition_key_parts * "-").to_s[/.+/m] || "-")
+    end
+
+    begin
+      @codec.encode(event)
+    rescue => e
+      @logger.warn("Error encoding event", :exception => e, :event => event)
+    end
+  end
+
+  public
+  def close
+    @producer.flushSync()
+    @producer.destroy()
+  end
+
+  def create_kpl_config
+    config = KPL.KinesisProducerConfiguration::new()
+
+    credentials_provider = create_credentials_provider
+    metrics_credentials_provider = create_metrics_credentials_provider
+
+    config.setAggregationEnabled(@aggregation_enabled)
+    config.setAggregationMaxCount(@aggregation_max_count)
+    config.setAggregationMaxSize(@aggregation_max_size)
+    config.setCloudwatchEndpoint(@cloudwatch_endpoint) if !@cloudwatch_endpoint.nil?
+    config.setCloudwatchPort(@cloudwatch_port)
+    config.setCollectionMaxCount(@collection_max_count)
+    config.setCollectionMaxSize(@collection_max_size)
+    config.setConnectTimeout(@connect_timeout)
+    config.setCredentialsProvider(credentials_provider)
+    config.setCredentialsRefreshDelay(@credentials_refresh_delay)
+    config.setEnableCoreDumps(@enable_core_dumps)
+    config.setFailIfThrottled(@fail_if_throttled)
+    config.setLogLevel(@log_level)
+    config.setKinesisEndpoint(@kinesis_endpoint) if !@kinesis_endpoint.nil?
+    config.setKinesisPort(@kinesis_port)
+    config.setMaxConnections(@max_connections)
+    config.setMetricsCredentialsProvider(metrics_credentials_provider)
+    config.setMetricsGranularity(@metrics_granularity)
+    config.setMetricsLevel(@metrics_level)
+    config.setMetricsNamespace(@metrics_namespace)
+    config.setMetricsUploadDelay(@metrics_upload_delay)
+    config.setMinConnections(@min_connections)
+    config.setNativeExecutable(@native_executable) if !@native_executable.nil?
+    config.setRateLimit(@rate_limit)
+    config.setRecordMaxBufferedTime(@record_max_buffered_time)
+    config.setRecordTtl(@record_ttl)
+    config.setRegion(@region)
+    config.setRequestTimeout(@request_timeout)
+    config.setTempDirectory(@temp_directory) if !@temp_directory.nil?
+    config.setVerifyCertificate(@verify_certificate)
+
+    config
+  end
+
+  def create_sts_provider(base_provider, arn)
+    client_config = com.amazonaws.ClientConfiguration.new()
+    if @sts_proxy_host
+      client_config.setProxyHost(@sts_proxy_host)
+    end
+    if @sts_proxy_port
+      client_config.setProxyPort(@sts_proxy_port)
+    end
+    provider = AWSAuth.STSAssumeRoleSessionCredentialsProvider.new(
+      base_provider, arn, "logstash-output-kinesis", client_config)
+    provider
+  end
+
+  def create_credentials_provider
+    provider = AWSAuth.DefaultAWSCredentialsProviderChain.new()
+    if @access_key and @secret_key
+      provider = BasicKinesisCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@access_key, @secret_key))
+    end
+    if @role_arn
+      provider = create_sts_provider(provider, @role_arn)
+    end
+    provider
+  end
+
+  def create_metrics_credentials_provider
+    provider = AWSAuth.DefaultAWSCredentialsProviderChain.new()
+    if @metrics_access_key and @metrics_secret_key
+      provider = BasicKinesisCredentialsProvider.new(AWSAuth.BasicAWSCredentials.new(@metrics_access_key, @metrics_secret_key))
+    end
+    if @metrics_role_arn
+      provider = create_sts_provider(provider, @metrics_role_arn)
+    end
+    provider
+  end
+
+  def send_record(event, payload)
+    begin
+      event_blob = ByteBuffer::wrap(payload.to_java_bytes)
+      @producer.addUserRecord(event.sprintf(@stream_name), event.get("[@metadata][partition_key]"), event_blob)
+    rescue => e
+      @logger.warn("Error writing event to Kinesis", :exception => e)
+    end
+
+    num = @producer.getOutstandingRecordsCount()
+    if num > @max_pending_records
+      @logger.warn("Kinesis is too busy - blocking until things have cleared up")
+      @producer.flushSync()
+      @logger.info("Okay - I've stopped blocking now")
+    end
+  end
+end
+
+class BasicKinesisCredentialsProvider
+  java_implements 'com.amazonaws.auth.AWSCredentialsProvider'
+
+  def initialize(credentials)
+    @credentials = credentials
+  end
+
+  java_signature 'com.amazonaws.auth.AWSCredentials getCredentials()'
+  def getCredentials
+    @credentials
+  end
+
+  java_signature 'void refresh()'
+  def refresh
+    # Noop.
+  end
+end

data/lib/logstash-output-kinesis/version.rb

@@ -0,0 +1,3 @@
+module LogstashOutputKinesis
+  VERSION = "5.1.3"
+end

data/lib/logstash-output-kinesis_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/logstash-output-kinesis.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "logstash-output-kinesis/version"
+
+Gem::Specification.new do |s|
+  s.name = "logstash-output-kinesis-jdk11"
+  s.version = LogstashOutputKinesis::VERSION
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "This output plugin sends records to Kinesis using the Kinesis Producer Library (KPL)"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis. This gem is not a stand-alone program"
+  s.authors = ["Sam Day"]
+  s.email = "me@samcday.com.au"
+  s.homepage = "https://www.github.com/samcday/logstash-output-kinesis"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {
+    "logstash_plugin" => "true",
+    "logstash_group" => "output"
+  }
+
+  s.platform = "java"
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 5.0.0"
+  s.add_runtime_dependency "logstash-codec-plain", "< 6.0.0"
+  s.add_runtime_dependency "logstash-codec-json", "< 6.0.0"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/kinesis_spec.rb

@@ -0,0 +1,96 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/kinesis"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe LogStash::Outputs::Kinesis do
+  let(:config) {{
+    "stream_name" => "test",
+    "region" => "ap-southeast-2",
+    "metrics_level" => "none"
+  }}
+  let(:sample_event) {
+    LogStash::Event.new({
+      "message" => "hello",
+      'stream_name' => 'my_stream',
+      "field1"  => "foo",
+      "field2"  => "bar"
+    })
+  }
+
+  KPL = com.amazonaws.services.kinesis.producer
+
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "kinesis").new(config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate config with default values' do
+      output = LogStash::Outputs::Kinesis.new(config)
+      insist { output.randomized_partition_key } == false
+      insist { output.event_partition_keys } == []
+    end
+  end
+
+  context "when receiving message" do
+    it "sends record to Kinesis" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+
+      output = LogStash::Outputs::Kinesis.new (config)
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support Event#sprintf placeholders in stream_name" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with("my_stream", anything, anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "stream_name" => "%{stream_name}",
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support blank partition keys" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, "-", anything)
+
+      output = LogStash::Outputs::Kinesis.new(config)
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support randomized partition keys" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, /[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+-[0-9a-f]+/, anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "randomized_partition_key" => true
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+
+    it "should support fixed partition keys" do
+      # the partition key ends up being an empty string plus the first field
+      # we choose joined by a hyphen. this is a holdover from earlier versions
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+        .with(anything, "-foo", anything)
+
+      output = LogStash::Outputs::Kinesis.new(config.merge({
+        "event_partition_keys" => ["[field1]", "[field2]"]
+      }))
+      output.register
+      output.receive(sample_event)
+      output.close
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,91 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+
+    # ... except this breaks mocking on a Java class it seems.
+    mocks.verify_partial_doubles = false
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Limits the available syntax to the non-monkey patched syntax that is recommended.
+  # For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-kinesis-jdk11
+version: !ruby/object:Gem::Version
+  version: 5.1.3
+platform: java
+authors:
+- Sam Day
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis.
+  This gem is not a stand-alone program
+email: me@samcday.com.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".editorconfig"
+- ".gemrelease"
+- ".gitattributes"
+- ".github_changelog_generator"
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- build.gradle
+- lib/logstash-output-kinesis/version.rb
+- lib/logstash-output-kinesis_jars.rb
+- lib/logstash/outputs/kinesis.rb
+- logstash-output-kinesis.gemspec
+- spec/outputs/kinesis_spec.rb
+- spec/spec_helper.rb
+- vendor/jar-dependencies/runtime-jars/amazon-kinesis-producer-0.12.6.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-core-1.11.128.jar
+- vendor/jar-dependencies/runtime-jars/aws-java-sdk-sts-1.11.128.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-io-2.4.jar
+- vendor/jar-dependencies/runtime-jars/commons-lang-2.6.jar
+- vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
+- vendor/jar-dependencies/runtime-jars/guava-18.0.jar
+- vendor/jar-dependencies/runtime-jars/httpclient-4.5.2.jar
+- vendor/jar-dependencies/runtime-jars/httpcore-4.4.4.jar
+- vendor/jar-dependencies/runtime-jars/ion-java-1.0.2.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.6.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/jackson-dataformat-cbor-2.6.6.jar
+- vendor/jar-dependencies/runtime-jars/jaxb-api-2.4.0-b180830.0359.jar
+- vendor/jar-dependencies/runtime-jars/jmespath-java-1.11.128.jar
+- vendor/jar-dependencies/runtime-jars/joda-time-2.8.1.jar
+- vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/protobuf-java-2.6.1.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
+homepage: https://www.github.com/samcday/logstash-output-kinesis
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
+specification_version: 4
+summary: This output plugin sends records to Kinesis using the Kinesis Producer Library
+  (KPL)
+test_files:
+- spec/outputs/kinesis_spec.rb
+- spec/spec_helper.rb