fluent-plugin-kinesis 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d0cf5075ce9585dd683ff3b5fe9fc2e26bcd3794
-  data.tar.gz: 59fd9d0ee641c986fcadf152a16d575d7b63ea8d
+  metadata.gz: 6dc44b8f1a4d7220c6d4c20dd4e33baa5ed70a78
+  data.tar.gz: 2d012e6a2733199c34308a4469743b4bf8c957bb
 SHA512:
-  metadata.gz: c7a95d98983572dc8818c97cfe3f0556ac441e95d7e7258d976b64aa46df5d0e3ed6d4c5c921aee73c2b4db9e5ee2b93a6ad8b413af4bc883e943d3fbbd74fc7
-  data.tar.gz: f3c6c7ee1f9163f23383b15240d8c84552a6d505aed304a49ed785c96e891a1ce1398ba9de98f0e96a351ba9e3b9485e0755c7548da6c3f7e3daf83cde35d28d
+  metadata.gz: f5cdbc4d830d1c71b4df65feaf3ec246434768863d4e533e4bd4bf6c10509f67ed055e3112de3476157372e28016f53fa140e50ad5cd323e65cc2d274ccb4d6d
+  data.tar.gz: 969d010695643a4e94abc934fd877fb7bc96991d77ea596d23c749a4a89aa4b7263e1606dfb37cb32fa4ea664b6839f1b1c5e3482fead17d32304f8b97cfb94d

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.4.0
+- Feature - Add option to ensure Kinesis Stream connection. [#35](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/35)
+- Feature - Add option to support zlib compression. [#39](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/39)
+
+Note: We introduced [Semantic Versioning](http://semver.org/) here.
+
 ## 0.3.6
 
 - **Cross account access support**: Added support for cross account access for Amazon Kinesis stream. With this update, you can put reocrds to streams those are owned by other AWS account. This feature is achieved by [AssumeRole](http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html).

data/README.md

@@ -119,6 +119,10 @@ It should be in form like "us-east-1", "us-west-2".
 Refer to [Regions and Endpoints in AWS General Reference](http://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region)
 for supported regions.
 
+### ensure_stream_connection
+
+When enabled, the plugin checks and ensures a connection to the stream you are using by [DescribeStream](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html) and throws exception if it fails. Enabled by default.
+
 ### http_proxy
 
 Proxy server, if any.
@@ -196,13 +200,19 @@ Integer, default is 3. When **order_events** is false, the plugin will put multi
 records to Amazon Kinesis in batches using PutRecords. A set of records in a batch
 may fail for reasons documented in the Kinesis Service API Reference for PutRecords.
 Failed records will be retried **retries_on_putrecords** times. If a record
-fails all retries an error log will be emitted. 
+fails all retries an error log will be emitted.
 
 ### use_yajl
 
 Boolean, default is false.
 In case you find error `Encoding::UndefinedConversionError` with multibyte texts, you can avoid that error with this option.
 
+### zlib_compression
+
+Boolean, default is false.
+Zlib compresses the message data blob.
+Each zlib compressed message must remain within megabyte in size.
+
 ### debug
 
 Boolean. Enable if you need to debug Amazon Kinesis API call. Default is false.

data/lib/fluent/plugin/out_kinesis.rb

@@ -17,6 +17,7 @@ require 'yajl'
 require 'logger'
 require 'securerandom'
 require 'fluent/plugin/version'
+require 'zlib'
 
 module FluentPluginKinesis
   class OutputFilter < Fluent::BufferedOutput
@@ -41,6 +42,7 @@ module FluentPluginKinesis
     # The 'region' parameter is optional because
     # it may be set as an environment variable.
     config_param :region,      :string, default: nil
+    config_param :ensure_stream_connection, :bool, default: true
 
     config_param :profile,          :string, :default => nil
     config_param :credentials_path, :string, :default => nil
@@ -56,6 +58,7 @@ module FluentPluginKinesis
     config_param :order_events,           :bool,   default: false
     config_param :retries_on_putrecords,  :integer, default: 3
     config_param :use_yajl,               :bool,   default: false
+    config_param :zlib_compression,       :bool,   default: false
 
     config_param :debug, :bool, default: false
 
@@ -102,7 +105,9 @@ module FluentPluginKinesis
       detach_multi_process do
         super
         load_client
-        check_connection_to_stream
+        if @ensure_stream_connection
+          check_connection_to_stream
+        end
       end
     end
 
@@ -120,15 +125,15 @@ module FluentPluginKinesis
     end
 
     def write(chunk)
-      data_list = chunk.to_enum(:msgpack_each).find_all{|record|
-        unless record_exceeds_max_size?(record['data'])
+      data_list = chunk.to_enum(:msgpack_each).map{|record|
+        build_data_to_put(record)
+      }.find_all{|record|
+        unless record_exceeds_max_size?(record[:data])
           true
         else
-          log.error sprintf('Record exceeds the %.3f KB(s) per-record size limit and will not be delivered: %s', PUT_RECORD_MAX_DATA_SIZE / 1024.0, record['data'])
+          log.error sprintf('Record exceeds the %.3f KB(s) per-record size limit and will not be delivered: %s', PUT_RECORD_MAX_DATA_SIZE / 1024.0, record[:data])
           false
         end
-      }.map{|record|
-        build_data_to_put(record)
       }
 
       if @order_events
@@ -220,7 +225,11 @@ module FluentPluginKinesis
     end
 
     def build_data_to_put(data)
-      Hash[data.map{|k, v| [k.to_sym, v] }]
+      if @zlib_compression
+        Hash[data.map{|k, v| [k.to_sym, k=="data" ? Zlib::Deflate.deflate(v) : v] }]
+      else
+        Hash[data.map{|k, v| [k.to_sym, v] }]
+      end
     end
 
     def put_record_for_order_events(data_list)

data/lib/fluent/plugin/version.rb

@@ -12,5 +12,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '0.3.6'
+  VERSION = '0.4.0'
 end

data/test/plugin/test_out_kinesis.rb

@@ -30,6 +30,14 @@ class KinesisOutputTest < Test::Unit::TestCase
     use_yajl true
   ]
 
+  CONFIG_WITH_COMPRESSION = CONFIG + %[
+    zlib_compression true
+  ]
+
+  CONFIG_YAJL_WITH_COMPRESSION = CONFIG_YAJL + %[
+    zlib_compression true
+  ]
+
   def create_driver(conf = CONFIG, tag='test')
     Fluent::Test::BufferedOutputTestDriver
       .new(FluentPluginKinesis::OutputFilter, tag).configure(conf)
@@ -143,6 +151,7 @@ class KinesisOutputTest < Test::Unit::TestCase
     conf = %[
       stream_name test_stream
       region us-east-1
+      ensure_stream_connection false
       http_proxy http://proxy:3333/
       partition_key test_partition_key
       partition_key_expr record
@@ -154,6 +163,7 @@ class KinesisOutputTest < Test::Unit::TestCase
     d = create_driver(conf)
     assert_equal 'test_stream', d.instance.stream_name
     assert_equal 'us-east-1', d.instance.region
+    assert_equal false, d.instance.ensure_stream_connection
     assert_equal 'http://proxy:3333/', d.instance.http_proxy
     assert_equal 'test_partition_key', d.instance.partition_key
     assert_equal 'Proc',
@@ -237,7 +247,7 @@ class KinesisOutputTest < Test::Unit::TestCase
 
 
   data("json"=>CONFIG, "yajl"=>CONFIG_YAJL)
-  def test_format(config)
+  def test_format_without_compression(config)
 
     d = create_driver(config)
 
@@ -276,6 +286,46 @@ class KinesisOutputTest < Test::Unit::TestCase
     d.run
   end
 
+  data("json"=>CONFIG_WITH_COMPRESSION, "yajl"=>CONFIG_YAJL_WITH_COMPRESSION)
+  def test_format_with_compression(config)
+
+    d = create_driver(config)
+
+    data1 = {"test_partition_key"=>"key1","a"=>1,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+    data2 = {"test_partition_key"=>"key2","a"=>2,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit(data1, time)
+    d.emit(data2, time)
+
+    d.expect_format({
+      'data' => data1.to_json,
+      'partition_key' => 'key1' }.to_msgpack
+    )
+    d.expect_format({
+      'data' => data2.to_json,
+      'partition_key' => 'key2' }.to_msgpack
+    )
+
+    client = create_mock_client
+    client.describe_stream(stream_name: 'test_stream')
+    client.put_records(
+      stream_name: 'test_stream',
+      records: [
+        {
+          data: Zlib::Deflate.deflate(data1.to_json),
+          partition_key: 'key1'
+        },
+        {
+          data: Zlib::Deflate.deflate(data2.to_json),
+          partition_key: 'key2'
+        }
+      ]
+    ) { {} }
+
+    d.run
+  end
+
   def test_order_events
 
     d = create_driver(CONFIG + "\norder_events true")
@@ -349,7 +399,7 @@ class KinesisOutputTest < Test::Unit::TestCase
     )
   end
 
-  def test_multibyte_with_yajl
+  def test_multibyte_with_yajl_without_compression
 
     d = create_driver(CONFIG_YAJL)
 
@@ -380,6 +430,37 @@ class KinesisOutputTest < Test::Unit::TestCase
     d.run
   end
 
+  def test_multibyte_with_yajl_with_compression
+
+    d = create_driver(CONFIG_YAJL_WITH_COMPRESSION)
+
+    data1 = {"test_partition_key"=>"key1","a"=>"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB","time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+    json = Yajl.dump(data1)
+    data1["a"].force_encoding("ASCII-8BIT")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit(data1, time)
+
+    d.expect_format({
+      'data' => json,
+      'partition_key' => 'key1' }.to_msgpack
+    )
+
+    client = create_mock_client
+    client.describe_stream(stream_name: 'test_stream')
+    client.put_records(
+      stream_name: 'test_stream',
+      records: [
+        {
+          data: Zlib::Deflate.deflate(json),
+          partition_key: 'key1'
+        }
+      ]
+    ) { {} }
+
+    d.run
+  end
+
   def test_get_key
     d = create_driver
     assert_equal(

metadata

@@ -1,123 +1,123 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-08 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.53
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.53
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '0.13'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.12
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.12
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.8
 description: 
@@ -126,8 +126,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTORS.txt
 - Gemfile
@@ -150,12 +150,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []