fluent-plugin-kinesis 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e552b291a04585897a47473fab2aafb95b7b3bc1
-  data.tar.gz: b48ce1dc2e7359f39d1ce43f303aceca35755bd1
+  metadata.gz: 1afe44290b2075c66e362e31b6ad9239b8b8a4f1
+  data.tar.gz: 8317862b2882294c24efd9d1e1d3434e37a82ba6
 SHA512:
-  metadata.gz: 828c58279188411787abb483731339458579c8fcd869cc122cbfd4edea6402749cc0ec6b0f13844534ba7214297983383073a0cde6a2a8965d67207f9e6647d7
-  data.tar.gz: a8b3826baa130ccd2470fb1c91f5df603eb937e7f789a1d8357fcdb1d9a7c74643597c6b355ad5e48ee05edc8ed8b372336e88338b06e03088b9c2cf6f7ac43a
+  metadata.gz: d3ed7132f9ab1022c75de1eef2732858adf4cacfeb607b4a7543e3989ce6541d14b601521062eedd966bd49f52a3f8d5812505e1fedcae6fd098c29ec863f35e
+  data.tar.gz: 890a373bdb0c9752026fc52120b444b7eb84dbf3abc116ea4d8d78816cb0dcf8f5f35c83ca888ab295dc251a4599e1d2f9681f7ee892c0ccf93be761409a893c

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 0.3.4
+
+- **Multi-byte UTF-8 support**: We now support multi-byte UTF-8 by using *use_yajl* option.
+
 ## 0.3.3
 
 - **Security improvements**: Disabled logging `aws_key_id` and `aws_sec_key` into log file.

data/README.md

@@ -186,6 +186,11 @@ may fail for reasons documented in the Kinesis Service API Reference for PutReco
 Failed records will be retried **retries_on_putrecords** times. If a record
 fails all retries an error log will be emitted. 
 
+### use_yajl
+
+Boolean, default is false.
+In case you find error `Encoding::UndefinedConversionError` with multibyte texts, you can avoid that error with this option.
+
 ### debug
 
 Boolean. Enable if you need to debug Amazon Kinesis API call. Default is false.

data/lib/fluent/plugin/out_kinesis.rb

@@ -13,6 +13,7 @@
 
 require 'aws-sdk-core'
 require 'multi_json'
+require 'yajl'
 require 'logger'
 require 'securerandom'
 require 'fluent/plugin/version'
@@ -49,6 +50,7 @@ module FluentPluginKinesis
     config_param :explicit_hash_key_expr, :string, default: nil
     config_param :order_events,           :bool,   default: false
     config_param :retries_on_putrecords,  :integer, default: 3
+    config_param :use_yajl,               :bool,   default: false
 
     config_param :debug, :bool, default: false
 
@@ -87,6 +89,8 @@ module FluentPluginKinesis
         )
         @explicit_hash_key_proc = eval(explicit_hash_key_proc_str)
       end
+
+      @dump_class = @use_yajl ? Yajl : JSON
     end
 
     def start
@@ -99,7 +103,7 @@ module FluentPluginKinesis
 
     def format(tag, time, record)
       data = {
-        data: record.to_json,
+        data: @dump_class.dump(record),
         partition_key: get_key(:partition_key,record)
       }
 
@@ -261,7 +265,7 @@ module FluentPluginKinesis
             log.error sprintf(
               'Could not put record, Error: %s, Record: %s',
               record[:error_code],
-              record[:body].to_json
+              @dump_class.dump(record[:body])
             )
           }
         end

data/lib/fluent/plugin/version.rb

@@ -12,5 +12,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '0.3.3'
+  VERSION = '0.3.4'
 end

data/test/plugin/test_out_kinesis.rb

@@ -26,6 +26,10 @@ class KinesisOutputTest < Test::Unit::TestCase
     partition_key test_partition_key
   ]
 
+  CONFIG_YAJL= CONFIG + %[
+    use_yajl true
+  ]
+
   def create_driver(conf = CONFIG, tag='test')
     Fluent::Test::BufferedOutputTestDriver
       .new(FluentPluginKinesis::OutputFilter, tag).configure(conf)
@@ -57,6 +61,7 @@ class KinesisOutputTest < Test::Unit::TestCase
       explicit_hash_key test_hash_key
       explicit_hash_key_expr record
       order_events true
+      use_yajl true
     ]
     d = create_driver(conf)
     assert_equal 'test_stream', d.instance.stream_name
@@ -74,6 +79,7 @@ class KinesisOutputTest < Test::Unit::TestCase
       d.instance.instance_variable_get(:@explicit_hash_key_proc).call('a')
     assert_equal true, d.instance.order_events
     assert_equal nil, d.instance.instance_variable_get(:@sequence_number_for_ordering)
+    assert_equal true, d.instance.use_yajl
   end
 
   def test_mode_configuration
@@ -141,9 +147,11 @@ class KinesisOutputTest < Test::Unit::TestCase
 
   end
 
-  def test_format
 
-    d = create_driver
+  data("json"=>CONFIG, "yajl"=>CONFIG_YAJL)
+  def test_format(config)
+
+    d = create_driver(config)
 
     data1 = {"test_partition_key"=>"key1","a"=>1,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
     data2 = {"test_partition_key"=>"key2","a"=>2,"time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
@@ -253,6 +261,37 @@ class KinesisOutputTest < Test::Unit::TestCase
     )
   end
 
+  def test_multibyte_with_yajl
+
+    d = create_driver(CONFIG_YAJL)
+
+    data1 = {"test_partition_key"=>"key1","a"=>"\xE3\x82\xA4\xE3\x83\xB3\xE3\x82\xB9\xE3\x83\x88\xE3\x83\xBC\xE3\x83\xAB","time"=>"2011-01-02T13:14:15Z","tag"=>"test"}
+    json = Yajl.dump(data1)
+    data1["a"].force_encoding("ASCII-8BIT")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit(data1, time)
+
+    d.expect_format({
+      'data' => json,
+      'partition_key' => 'key1' }.to_msgpack
+    )
+
+    client = create_mock_client
+    client.describe_stream(stream_name: 'test_stream')
+    client.put_records(
+      stream_name: 'test_stream',
+      records: [
+        {
+          data: json,
+          partition_key: 'key1'
+        }
+      ]
+    ) { {} }
+
+    d.run
+  end
+
   def test_get_key
     d = create_driver
     assert_equal(

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler