logstash-output-kinesis 0.0.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 01f7b59b419f5a2db26f399e6cda575162ea423b
+  data.tar.gz: 9d4f13f770e6747a989dfdd4f26a80e95e813533
+SHA512:
+  metadata.gz: 72541dfdd7b6f2edd33180a80120937813c0089d0a6d23002039b65501d95b3eab15b036e2c19266cb29472b3cf84124ba825a5a687d8301ea413a22333b2979
+  data.tar.gz: cd3e49ab33931bb3abbba2a42cf41a46650962bee90ae84546824d0fa5a8bed56bbedf67e5875b86041952aa474034b17468a1e1fcf69b7bb3d45f5ffbe467db

data/.editorconfig

@@ -0,0 +1,11 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{rb,gemspec}]
+indent_style = space
+indent_size = 2

data/.gemrelease

@@ -0,0 +1,3 @@
+bump:
+  tag: true
+  release: true

data/.gitignore

@@ -0,0 +1,9 @@
+/lib/**/*.jar
+/lib/*_jars.rb
+/*.gem
+.Gemfile.lock
+.Gemfile.bak
+/.bundle
+/vendor
+.DS_Store
+/Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+jruby-1.7.17

data/.travis.yml

@@ -0,0 +1 @@
+language: ruby

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 0.0.1
+
+Initial version

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Sam Day <http://www.samcday.com.au>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,96 @@
+# Kinesis Output Plugin
+
+[![Build Status][badge-travis]][travis]
+[![Gem info][badge-gem]][rubygems]
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It will send log records to a [Kinesis stream](https://aws.amazon.com/kinesis/), using the [KPL](https://docs.aws.amazon.com/kinesis/latest/dev/developing-producers-with-kpl.html) library.
+
+
+## Configuration
+
+Minimum required configuration to get this plugin chugging along:
+
+```
+output {
+  kinesis {
+    stream_name => "logs-stream"
+    region => "ap-southeast-2"
+  }
+}
+```
+
+This plugin accepts a wide range of configuration options, most of which come from the underlying KPL library itself. [View the full list of KPL configuration options here.][kpldoc]
+
+Please note that configuration options are snake_cased instead of camelCased. So, where [KinesisProducerConfiguration][kpldoc] offers a `setMetricsLevel` option,this plugin accepts a `metrics_level` option.
+
+### AWS Credentials
+
+There aren't many Kinesis streams out there that allow you to write to them without some AWS credentials.
+
+This plugin does not allow you to specify credentials directly. Instead, the AWS SDK [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) is used. It's quite flexible,you can provide your credentials via any of the following mechanisms:
+
+ * `AWS_ACCESS_KEY_ID` / `AWS_SECRET_KEY` environment variables
+ * `~/.aws/credentials` credentials file
+ * Instance profile for your running EC2 instance
+
+### Building a partition key
+
+Kinesis demands a [partition key](https://docs.aws.amazon.com/kinesis/latest/dev/key-concepts.html#partition-key) be provided for each record. By default, this plugin will provide a very boring partition key of `-`. However, you can configure it to compute a partition key from fields in your log events.
+
+```
+output {
+  kinesis {
+    # ...
+    event_partition_keys => ["[field1]", "[field2]"]
+  }
+}
+```
+
+
+## Known Issues
+
+### Noisy shutdown
+
+During shutdown of Logstash, you might get noisy warnings like this:
+
+```
+[pool-1-thread-6] WARN com.amazonaws.services.kinesis.producer.Daemon - Exception during updateCredentials
+java.lang.InterruptedException: sleep interrupted
+at java.lang.Thread.sleep(Native Method)
+at com.amazonaws.services.kinesis.producer.Daemon$5.run(Daemon.java:316)
+at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
+at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
+at java.util.concurrent.FutureTask.run(FutureTask.java:166)
+at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
+at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
+at java.lang.Thread.run(Thread.java:724)
+```
+
+This is caused by [amazon-kinesis-producer#10](https://github.com/awslabs/amazon-kinesis-producer/issues/10)
+
+
+## Developing
+
+```sh
+bundle install
+bundle exec rake
+```
+
+
+## Contributions
+
+Are more than welcome. Raising an issue is great, raising a PR is better, raising a PR with tests is best.
+
+
+## License
+
+[Apache License 2.0](LICENSE)
+
+[travis]: https://travis-ci.org/samcday/logstash-output-kinesis
+[rubygems]: https://rubygems.org/gems/logstash-output-kinesis
+[kpldoc]: https://github.com/awslabs/amazon-kinesis-producer/blob/v0.10.0/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38
+
+[badge-travis]: https://img.shields.io/travis/samcday/logstash-output-kinesis.svg?style=flat-square
+[badge-gem]: https://img.shields.io/gem/v/logstash-output-kinesis.svg?style=flat-square

data/Rakefile

@@ -0,0 +1,8 @@
+require "logstash/devutils/rake"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = Dir.glob('spec/**/*_spec.rb')
+end
+
+task :default => :spec

data/lib/logstash-output-kinesis/version.rb

@@ -0,0 +1,3 @@
+module LogstashOutputKinesis
+  VERSION = "0.0.2"
+end

data/lib/logstash/outputs/kinesis.rb

@@ -0,0 +1,122 @@
+# encoding: utf-8
+
+require "java"
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash-output-kinesis_jars"
+
+# Sends log events to a Kinesis stream. This output plugin uses the official Amazon KPL.
+# Most of the configuration options in this plugin are simply passed on to
+# link:https://github.com/awslabs/amazon-kinesis-producer/blob/v0.10.0/java/amazon-kinesis-producer/src/main/java/com/amazonaws/services/kinesis/producer/KinesisProducerConfiguration.java#L38[KinesisProducerConfiguration]
+class LogStash::Outputs::Kinesis < LogStash::Outputs::Base
+  config_name "kinesis"
+
+  default :codec, 'json'
+
+  # The name of the stream to send data to.
+  config :stream_name, :validate => :string, :required => true
+  # A list of event data keys to use when constructing a partition key
+  config :event_partition_keys, :validate => :array, :default => []
+
+  config :aggregation_enabled, :validate => :boolean, :default => true
+  config :aggregation_max_count, :validate => :number, :default => 4294967295
+  config :aggregation_max_size, :validate => :number, :default => 51200
+  config :collection_max_count, :validate => :number, :default => 500
+  config :collection_max_size, :validate => :number, :default => 5242880
+  config :connect_timeout, :validate => :number, :default => 6000
+  config :credentials_refresh_delay, :validate => :number, :default => 5000
+  config :custom_endpoint, :validate => :string, :default => nil
+  config :fail_if_throttled, :validate => :boolean, :default => false
+  config :log_level, :validate => :string, :default => "info"
+  config :max_connections, :validate => :number, :default => 4
+  config :metrics_granularity, :validate => ["global", "stream", "shard"], :default => "shard"
+  config :metrics_level, :validate => ["none", "summary", "detailed"], :default => "detailed"
+  config :metrics_namespace, :validate => :string, :default => "KinesisProducerLibrary"
+  config :metrics_upload_delay, :validate => :number, :default => 60000
+  config :min_connections, :validate => :number, :default => 1
+  config :native_executable, :validate => :string, :default => nil
+  config :port, :validate => :number, :default => 443
+  config :rate_limit, :validate => :number, :default => 150
+  config :record_max_buffered_time, :validate => :number, :default => 100
+  config :record_ttl, :validate => :number, :default => 30000
+  config :region, :validate => :string, :required => true
+  config :request_timeout, :validate => :number, :default => 6000
+  config :temp_directory, :validate => :string, :default => nil
+  config :verify_certificate, :validate => :boolean, :default => true
+
+  KPL = com.amazonaws.services.kinesis.producer
+  ByteBuffer = java.nio.ByteBuffer
+
+  public
+  def register
+    @producer = KPL.KinesisProducer::new(create_kpl_config)
+    @codec.on_event(&method(:send_record))
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    # Haha - gawd. If I don't put an empty string in the array, then calling .join()
+    # on it later will result in a US-ASCII string if the array is empty. Ruby is awesome.
+    partition_key_parts = [""]
+
+    @event_partition_keys.each do |partition_key_name|
+      if not event[partition_key_name].nil? and event[partition_key_name].length > 0
+        partition_key_parts << event[partition_key_name].to_s
+        break
+      end
+    end
+
+    event["[@metadata][partition_key]"] = (partition_key_parts * "-").to_s[/.+/m] || "-"
+
+    begin
+      @codec.encode(event)
+    rescue => e
+      @logger.warn("Error encoding event", :exception => e, :event => event)
+    end
+  end
+
+  public
+  def teardown
+    @producer.flushSync()
+    @producer.destroy()
+  end
+
+  def create_kpl_config
+    config = KPL.KinesisProducerConfiguration::new()
+
+    config.setAggregationEnabled(@aggregation_enabled)
+    config.setAggregationMaxCount(@aggregation_max_count)
+    config.setAggregationMaxSize(@aggregation_max_size)
+    config.setCollectionMaxCount(@collection_max_count)
+    config.setCollectionMaxSize(@collection_max_size)
+    config.setConnectTimeout(@connect_timeout)
+    config.setCredentialsRefreshDelay(@credentials_refresh_delay)
+    config.setCustomEndpoint(@custom_endpoint) if !@custom_endpoint.nil?
+    config.setFailIfThrottled(@fail_if_throttled)
+    config.setLogLevel(@log_level)
+    config.setMaxConnections(@max_connections)
+    config.setMetricsGranularity(@metrics_granularity)
+    config.setMetricsLevel(@metrics_level)
+    config.setMetricsNamespace(@metrics_namespace)
+    config.setMetricsUploadDelay(@metrics_upload_delay)
+    config.setMinConnections(@min_connections)
+    config.setNativeExecutable(@native_executable) if !@native_executable.nil?
+    config.setPort(@port)
+    config.setRateLimit(@rate_limit)
+    config.setRecordMaxBufferedTime(@record_max_buffered_time)
+    config.setRecordTtl(@record_ttl)
+    config.setRegion(@region)
+    config.setRequestTimeout(@request_timeout)
+    config.setTempDirectory(@temp_directory) if !@temp_directory.nil?
+    config.setVerifyCertificate(@verify_certificate)
+
+    config
+  end
+
+  def send_record(event, payload)
+    event_blob = ByteBuffer::wrap(payload.to_java_bytes)
+    @producer.addUserRecord(@stream_name, event["[@metadata][partition_key]"], event_blob)
+  end
+end

data/logstash-output-kinesis.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+  s.name = "logstash-output-kinesis"
+  s.version = "0.0.1"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "This output plugin sends records to Kinesis using the Kinesis Producer Library (KPL)"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis. This gem is not a stand-alone program"
+  s.authors = ["Sam Day"]
+  s.email = "me@samcday.com.au"
+  s.homepage = "https://www.github.com/samcday/logstash-output-kinesis"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {
+    "logstash_plugin" => "true",
+    "logstash_group" => "output"
+  }
+
+  # Jar dependencies
+  s.requirements << "jar 'com.amazonaws:amazon-kinesis-producer', '0.10.0'"
+
+  s.platform = "java"
+
+  # Gem dependencies
+  s.add_runtime_dependency "jar-dependencies"
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "logstash-codec-json"
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "gem-release", "~>0.7.3"
+end

data/spec/outputs/kinesis_spec.rb

@@ -0,0 +1,27 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/kinesis"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe LogStash::Outputs::Kinesis do
+  let(:config) {{
+    "stream_name" => "test",
+    "region" => "ap-southeast-2",
+    "metrics_level" => "none"
+  }}
+  let(:sample_event) { LogStash::Event.new }
+
+  KPL = com.amazonaws.services.kinesis.producer
+
+  context "when receiving message" do
+    it "sends record to Kinesis" do
+      expect_any_instance_of(KPL::KinesisProducer).to receive(:addUserRecord)
+
+      output = LogStash::Outputs::Kinesis.new (config)
+      output.register
+      output.receive(sample_event)
+      output.teardown
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,91 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+
+    # ... except this breaks mocking on a Java class it seems.
+    mocks.verify_partial_doubles = false
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Limits the available syntax to the non-monkey patched syntax that is recommended.
+  # For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-kinesis
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: java
+authors:
+- Sam Day
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-07-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+  name: gem-release
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-output-kinesis. This gem is not a stand-alone program
+email: me@samcday.com.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .editorconfig
+- .gemrelease
+- .gitignore
+- .rspec
+- .ruby-version
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash-output-kinesis/version.rb
+- lib/logstash/outputs/kinesis.rb
+- logstash-output-kinesis.gemspec
+- spec/outputs/kinesis_spec.rb
+- spec/spec_helper.rb
+homepage: https://www.github.com/samcday/logstash-output-kinesis
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.amazonaws:amazon-kinesis-producer', '0.10.0'
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: This output plugin sends records to Kinesis using the Kinesis Producer Library (KPL)
+test_files:
+- spec/outputs/kinesis_spec.rb
+- spec/spec_helper.rb