fluent-plugin-kinesis 0.3.4 → 0.3.5

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## 0.3.5
+
+- **1MB record size limit**: Increased record size limit from 50KB to 1MB due to [Amazon Kinesis improvement.](http://aws.amazon.com/jp/about-aws/whats-new/2015/06/amazon-kinesis-announces-put-pricing-change-1mb-record-support-and-the-kinesis-producer-library/)
+- **Switching IAM user support**: Added support for [shared credential file](http://docs.aws.amazon.com/ja_jp/AWSSdkDocsRuby/latest/DeveloperGuide/prog-basics-creds.html#creds-specify-provider).
+
 ## 0.3.4
 
 - **Multi-byte UTF-8 support**: We now support multi-byte UTF-8 by using *use_yajl* option.

data/lib/fluent/plugin/out_kinesis.rb

@@ -28,7 +28,7 @@ module FluentPluginKinesis
     USER_AGENT_NAME = 'fluent-plugin-kinesis-output-filter'
     PROC_BASE_STR = 'proc {|record| %s }'
     PUT_RECORDS_MAX_COUNT = 500
-    PUT_RECORD_MAX_DATA_SIZE = 1024 * 50
+    PUT_RECORD_MAX_DATA_SIZE = 1024 * 1024
     PUT_RECORDS_MAX_DATA_SIZE = 1024 * 1024 * 5
 
     Fluent::Plugin.register_output('kinesis',self)
@@ -42,6 +42,9 @@ module FluentPluginKinesis
     # it may be set as an environment variable.
     config_param :region,      :string, default: nil
 
+    config_param :profile,          :string, :default => nil
+    config_param :credentials_path, :string, :default => nil
+
     config_param :stream_name,            :string
     config_param :random_partition_key,   :bool,   default: false
     config_param :partition_key,          :string, default: nil
@@ -119,7 +122,7 @@ module FluentPluginKinesis
         unless record_exceeds_max_size?(record['data'])
           true
         else
-          log.error sprintf('Record exceeds the 50KB per-record size limit and will not be delivered: %s', record['data'])
+          log.error sprintf('Record exceeds the %.3f KB(s) per-record size limit and will not be delivered: %s', PUT_RECORD_MAX_DATA_SIZE / 1024.0, record['data'])
           false
         end
       }.map{|record|
@@ -160,6 +163,11 @@ module FluentPluginKinesis
           access_key_id: @aws_key_id,
           secret_access_key: @aws_sec_key,
         )
+      elsif @profile
+        credentials_opts = {:profile_name => @profile}
+        credentials_opts[:path] = @credentials_path if @credentials_path
+        credentials = Aws::SharedCredentials.new(credentials_opts)
+        options[:credentials] = credentials
       end
 
       if @debug
@@ -226,15 +234,16 @@ module FluentPluginKinesis
       records_payload_length = 0
       data_list.each{|data_to_put|
         payload = data_to_put[:data]
-        if records.length >= PUT_RECORDS_MAX_COUNT or (records_payload_length + payload.length) >= PUT_RECORDS_MAX_DATA_SIZE
+        partition_key = data_to_put[:partition_key]
+        if records.length >= PUT_RECORDS_MAX_COUNT or (records_payload_length + payload.length + partition_key.length) >= PUT_RECORDS_MAX_DATA_SIZE
           records_array.push(records)
           records = []
           records_payload_length = 0
         end
         records.push(data_to_put)
-        records_payload_length += payload.length
+        records_payload_length += (payload.length + partition_key.length)
       }
-      records_array.push(records)
+      records_array.push(records) unless records.empty?
       records_array
     end
 

data/lib/fluent/plugin/version.rb

@@ -12,5 +12,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '0.3.4'
+  VERSION = '0.3.5'
 end

data/test/plugin/test_out_kinesis.rb

@@ -50,6 +50,66 @@ class KinesisOutputTest < Test::Unit::TestCase
     assert_equal 'test_partition_key', d.instance.partition_key
   end
 
+  def test_configure_with_credentials
+    d = create_driver(<<-EOS)
+      profile default
+      credentials_path /home/scott/.aws/credentials
+      stream_name test_stream
+      region us-east-1
+      partition_key test_partition_key
+    EOS
+
+    assert_equal 'default', d.instance.profile
+    assert_equal '/home/scott/.aws/credentials', d.instance.credentials_path
+    assert_equal 'test_stream', d.instance.stream_name
+    assert_equal 'us-east-1', d.instance.region
+    assert_equal 'test_partition_key', d.instance.partition_key
+  end
+
+  def test_load_client
+    client = stub(Object.new)
+    client.describe_stream
+    client.put_records { {} }
+
+    stub(Aws::Kinesis::Client).new do |options|
+      assert_equal("test_key_id", options[:access_key_id])
+      assert_equal("test_sec_key", options[:secret_access_key])
+      assert_equal("us-east-1", options[:region])
+      client
+    end
+
+    d = create_driver
+    d.run
+  end
+
+  def test_load_client_with_credentials
+    client = stub(Object.new)
+    client.describe_stream
+    client.put_records { {} }
+
+    stub(Aws::Kinesis::Client).new do |options|
+      assert_equal(nil, options[:access_key_id])
+      assert_equal(nil, options[:secret_access_key])
+      assert_equal("us-east-1", options[:region])
+
+      credentials = options[:credentials]
+      assert_equal("default", credentials.profile_name)
+      assert_equal("/home/scott/.aws/credentials", credentials.path)
+
+      client
+    end
+
+    d = create_driver(<<-EOS)
+      profile default
+      credentials_path /home/scott/.aws/credentials
+      stream_name test_stream
+      region us-east-1
+      partition_key test_partition_key
+    EOS
+
+    d.run
+  end
+
   def test_configure_with_more_options
 
     conf = %[
@@ -351,75 +411,99 @@ class KinesisOutputTest < Test::Unit::TestCase
 
   def test_record_exceeds_max_size
     d = create_driver
+
+    # PUT_RECORD_MAX_DATA_SIZE = 1024*1024 is way too big, try something smaller (10), just to verify the logic not the actual value
+    original_put_record_max_data_size = d.instance.class.send(:remove_const, :PUT_RECORD_MAX_DATA_SIZE) if d.instance.class.const_defined?(:PUT_RECORD_MAX_DATA_SIZE)
+    d.instance.class.const_set(:PUT_RECORD_MAX_DATA_SIZE, 10)
+
     string = ''
-    (1..1024).each{ string = string + '1' }
+    (1..5).each{ string = string + '1' }
     assert_equal(
       false,
       d.instance.send(:record_exceeds_max_size?,string)
     )
 
     string = ''
-    (1..(1024*50)).each{ string = string + '1' }
+    (1..10).each{ string = string + '1' }
     assert_equal(
       false,
       d.instance.send(:record_exceeds_max_size?,string)
     )
 
     string = ''
-    (1..(1024*51)).each{ string = string + '1' }
+    (1..11).each{ string = string + '1' }
     assert_equal(
       true,
       d.instance.send(:record_exceeds_max_size?,string)
     )
+
+    # reset the constant
+    d.instance.class.const_set(:PUT_RECORD_MAX_DATA_SIZE, original_put_record_max_data_size)
   end
 
   def test_build_records_array_to_put
     d = create_driver
 
+    # PUT_RECORDS_MAX_DATA_SIZE = 1024*1024*5 is way too big, try something smaller (100), just to verify the logic not the actual value
+    original_put_records_max_data_size = d.instance.class.send(:remove_const, :PUT_RECORDS_MAX_DATA_SIZE) if d.instance.class.const_defined?(:PUT_RECORDS_MAX_DATA_SIZE)
+    d.instance.class.const_set(:PUT_RECORDS_MAX_DATA_SIZE, 100)
+
+    # PUT_RECORDS_MAX_COUNT = 500 is way too big, try something smaller (10), just to verify the logic not the actual value
+    original_put_records_max_count = d.instance.class.send(:remove_const, :PUT_RECORDS_MAX_COUNT) if d.instance.class.const_defined?(:PUT_RECORDS_MAX_COUNT)
+    d.instance.class.const_set(:PUT_RECORDS_MAX_COUNT, 10)
+
     data_list = []
-    (0..500).each do |n|
-      data_list.push({data: n.to_s})
+    (0..10).each do |n|
+      data_list.push({data: '1', partition_key: '0'})
     end
     result = d.instance.send(:build_records_array_to_put,data_list)
     assert_equal(2,result.length)
-    assert_equal(500,result[0].length)
+    assert_equal(10,result[0].length)
     assert_equal(1,result[1].length)
 
     data_list = []
-    (0..1400).each do
-      data_list.push({data: '1'})
+    (0..24).each do
+      data_list.push({data: '1', partition_key: '0'})
     end
     result = d.instance.send(:build_records_array_to_put,data_list)
     assert_equal(3,result.length)
-    assert_equal(500,result[0].length)
-    assert_equal(500,result[1].length)
-    assert_equal(401,result[2].length)
+    assert_equal(10,result[0].length)
+    assert_equal(10,result[1].length)
+    assert_equal(5,result[2].length)
 
     data_list = []
-    data_string = ''
-    (0..(1024*30)).each do
-      data_string = data_string + '1'
-    end
-    (0..500).each do
-      data_list.push({data: data_string})
+    (0..20).each do
+      data_list.push({data: '0123456789', partition_key: '1'})
     end
+    # Should return 3 lists: 9*11 + 9*11 + 3*11
     result = d.instance.send(:build_records_array_to_put,data_list)
     assert_equal(3,result.length)
-    assert_equal(170,result[0].length)
+    assert_equal(9,result[0].length)
     assert_operator(
-      1024 * 1024 *5, :>,
-      result[0].reduce(0){|sum,i| sum + i[:data].length}
+      100, :>,
+      result[0].reduce(0){|sum,i| sum + i[:data].length + i[:partition_key].length}
     )
-    assert_equal(170,result[1].length)
+    assert_equal(9,result[1].length)
     assert_operator(
-      1024 * 1024 *5, :>,
-      result[1].reduce(0){|sum,i| sum + i[:data].length}
+      100, :>,
+      result[1].reduce(0){|sum,i| sum + i[:data].length + i[:partition_key].length}
     )
-    assert_equal(161,result[2].length)
+    assert_equal(3,result[2].length)
     assert_operator(
-      1024 * 1024 *5, :>,
-      result[2].reduce(0){|sum,i| sum + i[:data].length}
+      100, :>,
+      result[2].reduce(0){|sum,i| sum + i[:data].length + i[:partition_key].length}
     )
+
+    # reset the constants
+    d.instance.class.const_set(:PUT_RECORDS_MAX_DATA_SIZE, original_put_records_max_data_size)
+    d.instance.class.const_set(:PUT_RECORDS_MAX_COUNT, original_put_records_max_count)
+  end
+
+  def test_build_empty_array_to_put
+    d = create_driver
+    data_list = []
+    result = d.instance.send(:build_records_array_to_put,data_list)
+    assert_equal(0, result.length, 'Should return empty array if there is no record')
   end
 
   def test_build_data_to_put

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
+  prerelease: 
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-06 00:00:00.000000000 Z
+date: 2015-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -55,8 +62,9 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.53
     - - <
@@ -65,8 +73,9 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.53
     - - <
@@ -75,8 +84,9 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.0.12
     - - <
@@ -85,8 +95,9 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.0.12
     - - <
@@ -95,6 +106,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,6 +114,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -109,15 +122,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.5.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.5.8
 description: 
@@ -143,26 +158,27 @@ files:
 homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
 licenses:
 - Apache License, Version 2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd output plugin that sends events to Amazon Kinesis.
 test_files:
 - test/helper.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 1afe44290b2075c66e362e31b6ad9239b8b8a4f1
-  data.tar.gz: 8317862b2882294c24efd9d1e1d3434e37a82ba6
-SHA512:
-  metadata.gz: d3ed7132f9ab1022c75de1eef2732858adf4cacfeb607b4a7543e3989ce6541d14b601521062eedd966bd49f52a3f8d5812505e1fedcae6fd098c29ec863f35e
-  data.tar.gz: 890a373bdb0c9752026fc52120b444b7eb84dbf3abc116ea4d8d78816cb0dcf8f5f35c83ca888ab295dc251a4599e1d2f9681f7ee892c0ccf93be761409a893c