fluent-plugin-kinesis 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46273afe8f8e8fa0f70931b1f2bbf1893df30256
-  data.tar.gz: 55beac91c0b19714355e901ed256dbf16210a930
+  metadata.gz: 9f1802cbf044412c40fee66e52412c78a8f00c58
+  data.tar.gz: 3bb9b8bb9b5a079cbf3fa61f90b6b3e16cd81e77
 SHA512:
-  metadata.gz: 57f4784a5c00763fb358de48d74d161815a90a466f4e8afe895b91485bae1db6a6eba84b994def1ddf6a90f9a60742ac63b4abd91d1fa9eda16d360bb5c0db02
-  data.tar.gz: a5b6e7769c7ddce92bd9ee27e603c7bfbe41ac3a53d032b12b5edf30522a1eb98f70e54a75dee6ec33dd8a197f8af8effd3d373a0c98c9bb709e67ecd0d947a0
+  metadata.gz: acf1eefc4d2d071cb81cf22dd1969f5fe60276bfecd3616ac193b19106073c3d585bef51d3a5750f7a803a914e5a4cab75ea32d37153380d19b910c40891f35e
+  data.tar.gz: ceed8d7790f50d07256fc7a927ee053d3808d92ee6d444d87c96c30813283a8758b21d13364b476747efd777a1882dc729a0a7d3308777258a3214eb59f6c939

data/.travis.yml

@@ -8,6 +8,7 @@ rvm:
 gemfile:
   - Gemfile
   - gemfiles/Gemfile.fluentd-0.10.58
+  - gemfiles/Gemfile.fluentd-0.12
 
 os:
   - linux
@@ -22,5 +23,15 @@ sudo: false
 
 matrix:
   exclude:
+    - rvm: 2.0.0
+      os: osx
+    - rvm: 2.0.0
+      gemfile: Gemfile
+    - rvm: 2.1
+      os: osx
+      gemfile: gemfiles/Gemfile.fluentd-0.10.58
+    - rvm: 2.1
+      os: osx
+      gemfile: gemfiles/Gemfile.fluentd-0.12
     - rvm: 2.2
       os: osx

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 1.1.0
+
+- Feature - Derive stream name from fluentd tag for KPL [#67](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/67)
+- Enhancement - Make http_proxy parameter secret [#64](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/64)
+- Bug fix - Plugin incompatible with new fluentd release 0.14 [#70](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/70)
+- Misc - Fix legacy test and reduce travis tests [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74)
+- Misc - Some test, benchmark improvement [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74), [#75](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/75), [#76](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/76)
+
 ## 1.0.1
 
 - Bug fix - Instance profile credentials expiring [#58](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/58)

data/Makefile

@@ -12,26 +12,25 @@
 #  express or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
 
-.PHONY: install streams firehose producer dummer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
+.PHONY: test install benchmark benchmark-streams benchmark-producer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
 
 all:
 	bundle install
-	bundle exec rake
+	bundle exec rake binaries
+
+test:
+	bundle exec rake test
 
 install:
 	bundle exec rake install:local
 
-streams:
-	bundle exec fluentd -c benchmark/streams.conf -vv
-
-firehose:
-	bundle exec fluentd -c benchmark/firehose.conf -vv
+benchmark: benchmark-streams benchmark-producer
 
-producer:
-	bundle exec fluentd -c benchmark/producer.conf -vv
+benchmark-streams:
+	bundle exec rake benchmark TYPE=streams
 
-dummer:
-	bundle exec dummer -c benchmark/dummer.conf
+benchmark-producer:
+	bundle exec rake benchmark TYPE=producer
 
 hello:
 	echo Hello World | bundle exec fluent-cat --none dummy

data/README.md

@@ -527,15 +527,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
     git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
     cd aws-fluent-plugin-kinesis
     make # will install gems and download KPL jar file and extract binaries
-    make [stream/firehose/producer]
+    bundle exec fluentd -c /path/to/fluent.conf
 
-Then, in another terminal, run the command below. It will emit one record.
-
-    make hello
-
-Also, you can test streaming log data by `dummer`:
-
-    make dummer # keep writing to /tmp/dummy.log
+If you want to run benchmark, use `make benchmark`.
 
 ## Contributing
 

data/Rakefile

@@ -24,6 +24,7 @@ Rake::TestTask.new do |test|
 end
 
 load 'kinesis_producer/tasks/binary.rake'
-
 Rake::Task[:build].enhance [:binaries]
 Rake::Task[:test].enhance [:binaries]
+
+load 'benchmark/task.rake'

data/benchmark/task.rake

@@ -0,0 +1,72 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require_relative '../test/dummy_server'
+
+task :benchmark do
+  server = DummyServer.start
+  conf = profile_conf(ENV["TYPE"] || 'streams', ENV["RATE"] || 1000, server)
+  pid = spawn("bundle exec fluentd -i '#{conf}' -c benchmark/dummy.conf")
+  sleep 10
+  Process.kill("TERM", pid)
+  Process.wait
+  puts "Results: requets: #{server.requests.size}, raw_records: #{server.raw_records.size}, records: #{server.records.size}"
+  server.shutdown
+end
+
+def profile_conf(type, rate, server)
+  additional_conf = case type
+                  when 'streams', 'firehose'
+                    <<-EOS
+  endpoint https://localhost:#{server.port}
+  ssl_verify_peer false
+                    EOS
+                  when 'producer'
+                    <<-EOS
+  debug true
+  <kinesis_producer>
+    custom_endpoint localhost
+    port #{server.port}
+    verify_certificate false
+    record_max_buffered_time 1000
+    log_level error
+  </kinesis_producer>
+                    EOS
+                  end
+
+  conf = <<-EOS
+<source>
+  @type dummy
+  tag dummy
+  rate #{rate}
+</source>
+
+<match dummy>
+  @type kinesis_#{type}
+  flush_interval 1
+  buffer_chunk_limit 1m
+  try_flush_interval 0.1
+  queued_chunk_flush_interval 0.01
+  @log_level debug
+
+  num_threads 100
+
+  region ap-northeast-1
+  stream_name fluent-plugin-test
+
+#{additional_conf}
+</match>
+  EOS
+  conf
+end

data/fluent-plugin-kinesis.gemspec

@@ -46,7 +46,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug", ">= 3.3.0"
   spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
   spec.add_development_dependency "net-empty_port", ">= 0.0.2"
-  spec.add_development_dependency "dummer", ">= 0.4.0"
   spec.add_development_dependency "rubyzip", ">= 1.0.0"
   spec.add_development_dependency "mocha", ">= 1.1.0"
   spec.add_development_dependency "webmock", ">= 1.24.2"

data/gemfiles/Gemfile.fluentd-0.12

@@ -0,0 +1,20 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "~> 0.12.26"

data/lib/fluent/plugin/kinesis_helper.rb

@@ -20,6 +20,7 @@ module Fluent
   module KinesisHelper
     include Fluent::SetTimeKeyMixin
     include Fluent::SetTagKeyMixin
+    require 'fluent/process'
     include Fluent::DetachMultiProcessMixin
 
     def self.included(klass)

data/lib/fluent/plugin/kinesis_helper/class_methods.rb

@@ -45,9 +45,10 @@ module Fluent
 
       def config_param_for_producer
         const_set(:RequestType, :producer)
-        config_param :stream_name,   :string
-        config_param :region,        :string, default: nil
-        config_param :partition_key, :string, default: nil
+        config_param :stream_name,        :string, default: nil
+        config_param :stream_name_prefix, :string, default: nil
+        config_param :region,             :string, default: nil
+        config_param :partition_key,      :string, default: nil
         config_param_for_credentials
         config_param_for_format
         config_param_for_debug
@@ -69,7 +70,7 @@ module Fluent
       end
 
       def config_param_for_sdk
-        config_param :http_proxy,      :string, default: nil
+        config_param :http_proxy,      :string, default: nil, secret: true
         config_param :endpoint,        :string, default: nil
         config_param :ssl_verify_peer, :bool,   default: true
       end

data/lib/fluent/plugin/kinesis_helper/initialize.rb

@@ -32,6 +32,7 @@ module Fluent
             include API
           when kpl?
             require 'fluent/plugin/kinesis_helper/kpl'
+            require 'fluent/version'
             if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
               # Backport from https://github.com/fluent/fluentd/pull/757
               require 'fluent/plugin/patched_detach_process_impl'

data/lib/fluent/plugin/out_kinesis_producer.rb

@@ -20,6 +20,16 @@ module Fluent
     Fluent::Plugin.register_output('kinesis_producer', self)
     config_param_for_producer
 
+    def configure(conf)
+      super
+      unless @stream_name or @stream_name_prefix
+        raise Fluent::ConfigError, "'stream_name' or 'stream_name_prefix' is required"
+      end
+      if @stream_name and @stream_name_prefix
+        raise Fluent::ConfigError, "Only one of 'stream_name' or 'stream_name_prefix' is allowed"
+      end
+    end
+
     def write(chunk)
       records = convert_to_records(chunk)
       wait_futures(write_chunk_to_kpl(records))
@@ -31,7 +41,7 @@ module Fluent
       {
         data: data_format(tag, time, record),
         partition_key: key(record),
-        stream_name: @stream_name,
+        stream_name: @stream_name ? @stream_name : @stream_name_prefix + tag,
       }
     end
   end

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 #  permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-29 00:00:00.000000000 Z
+date: 2016-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -198,20 +198,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: dummer
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -288,12 +274,11 @@ files:
 - amazon-kinesis-producer-native-binaries/linux/kinesis_producer
 - amazon-kinesis-producer-native-binaries/osx/kinesis_producer
 - amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
-- benchmark/dummer.conf
-- benchmark/firehose.conf
-- benchmark/producer.conf
-- benchmark/streams.conf
+- benchmark/dummy.conf
+- benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.10.58
+- gemfiles/Gemfile.fluentd-0.12
 - lib/fluent/plugin/kinesis_helper.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
 - lib/fluent/plugin/kinesis_helper/class_methods.rb

data/benchmark/dummer.conf

@@ -1,13 +0,0 @@
-configure 'sample' do
-  output "/tmp/dummy.log"
-  rate 5000
-  delimiter "\t"
-  labeled true
-  field :id, type: :integer, countup: true, format: "%04d"
-  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
-  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
-  field :method, type: :string, any: %w[GET POST PUT]
-  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
-  field :reqtime, type: :float, range: 0.1..5.0
-  field :foobar, type: :string, length: 8
-end

data/benchmark/firehose.conf

@@ -1,24 +0,0 @@
-<source>
-  @type tail
-  path /tmp/dummy.log
-  format none
-  tag dummy
-</source>
-
-<source>
-  @type forward
-</source>
-
-<match dummy>
-  @type kinesis_firehose
-  flush_interval 1
-  buffer_chunk_limit 1m
-  try_flush_interval 0.1
-  queued_chunk_flush_interval 0.01
-  num_threads 15
-  detach_process 5
-  log_level debug
-
-  region us-west-2
-  delivery_stream_name fluent-plugin-test
-</match>

data/benchmark/producer.conf

@@ -1,28 +0,0 @@
-<source>
-  @type tail
-  path /tmp/dummy.log
-  format none
-  tag dummy
-</source>
-
-<source>
-  @type forward
-</source>
-
-<match dummy>
-  @type kinesis_producer
-  flush_interval 1
-  buffer_chunk_limit 1m
-  try_flush_interval 0.1
-  queued_chunk_flush_interval 0.01
-  num_threads 15
-  detach_process 5
-  log_level debug
-
-  region ap-northeast-1
-  stream_name fluent-plugin-test
-  debug true
-  <kinesis_producer>
-    record_max_buffered_time 1000
-  </kinesis_producer>
-</match>

data/benchmark/streams.conf

@@ -1,24 +0,0 @@
-<source>
-  @type tail
-  path /tmp/dummy.log
-  format none
-  tag dummy
-</source>
-
-<source>
-  @type forward
-</source>
-
-<match dummy>
-  @type kinesis_streams
-  flush_interval 1
-  buffer_chunk_limit 1m
-  try_flush_interval 0.1
-  queued_chunk_flush_interval 0.01
-  num_threads 15
-  detach_process 5
-  log_level debug
-
-  region ap-northeast-1
-  stream_name fluent-plugin-test
-</match>