fluent-plugin-td 0.10.23 → 0.10.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bea1cedfa710605a41e6621d3c1a5e735d30d41b
-  data.tar.gz: e674f3cc8a8f4ea2901b8232266bfe7cbbada432
+  metadata.gz: 46404732e76417f2f5110a8f97a219935c3c653c
+  data.tar.gz: ae4a4ba457f7c7bd5097fa37a9cc955bd02914bd
 SHA512:
-  metadata.gz: 256e1e8db03e0e8271a55e4d138d40d0c2af9f94cd9712d357625fd62bc87becbad890ec01ec649a90cd0390e103275a426a1b853e1d40c1fd5637ae88ab9f58
-  data.tar.gz: 02fb8daa4050de87b6ca2f2e7d87d5c732e9c75040c3588e893bd93255ae509dfa2fb48ed9850479c549b4ef55e85c8aa139d228a8237c7d06003145c69204c7
+  metadata.gz: e9812f2997140c7470641d82666ec59ec4bae779fb435f24fca1a821e0bdcb67184a0151b93e92cd130045b1223a352e96fe4d553e2017a3ea787e963f91c1b6
+  data.tar.gz: e332f546b28efe2446b7d820fe1f1fd9e74dd32bf2661c80640677b963fa194acba041e3597b22ef38fd91c4b14e2e0811f0d280c498fd64c0dd860cd512dc60

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.10.24 - 2015/02/05
+
+  * Change log level and message when a broken record is skipped to make it clear that it is skipped
+  * Skip non-hash record because plugin can't recover such situation
+  * Use MessagePack::Buffer to improve the performance with msgpack v0.5
+
+
 Release 0.10.23 - 2014/12/08
 
   * Relax fluentd version restriction

data/lib/fluent/plugin/out_tdlog.rb

@@ -183,9 +183,12 @@ module Fluent
     end
 
     def format_stream(tag, es)
-      out = ''
-      off = out.bytesize
+      out = $use_msgpack_5 ? MessagePack::Buffer.new : ''.force_encoding('ASCII-8BIT') # this condition will be removed after removed msgpack v0.4 support
+      off = out.size  # size is same as bytesize in ASCII-8BIT string
       es.each { |time, record|
+        # Applications may send non-hash record or broken chunk may generate non-hash record so such records should be skipped
+        next unless record.is_a?(Hash)
+
         begin
           if @anonymizes
             @anonymizes.each_pair { |key, scr|
@@ -205,8 +208,8 @@ module Fluent
           # TODO (a) Remove the transaction mechanism of fluentd
           #      or (b) keep transaction boundaries in in/out_forward.
           #      This code disables the transaction mechanism (a).
-          log.error "#{e}: #{summarize_record(record)}"
-          log.error_backtrace e.backtrace
+          log.warn "Skipped a broken record (#{e}): #{summarize_record(record)}"
+          log.warn_backtrace e.backtrace
           next
         end
 
@@ -216,7 +219,7 @@ module Fluent
           TreasureData::API.normalized_msgpack(record, out)
         end
 
-        noff = out.bytesize
+        noff = out.size
         sz = noff - off
         if sz > @record_size_limit
           # TODO don't raise error
@@ -225,7 +228,7 @@ module Fluent
         end
         off = noff
       }
-      out
+      out.to_s
     end
 
     def summarize_record(record)

data/lib/fluent/plugin/td_plugin_version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module TreasureDataPlugin
-    VERSION = '0.10.23'
+    VERSION = '0.10.24'
   end
 end

data/test/plugin/test_out_tdlog.rb

@@ -56,6 +56,25 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     assert_equal('TD1 testkey', @auth_header)
   end
 
+  def test_emit_with_broken_record
+    d = create_driver
+    time, records = stub_seed_values
+    records[1] = nil
+    records << 'string' # non-hash case
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table)
+
+    records.each { |record|
+      d.emit(record, time)
+    }
+    d.run
+
+    assert !d.instance.log.logs.any? { |line|
+      line =~ /undefined method/
+    }, 'nil record should be skipped'
+  end
+
   def test_emit_with_time_symbole
     d = create_driver
     time, records = stub_seed_values
@@ -99,7 +118,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
 
     assert_equal 0, d.emits.size
     assert d.instance.log.logs.select{ |line|
-      line =~ / \[error\]: Too many number of keys/
+      line =~ /Too many number of keys/
     }.size == 1, "too many keys error is not logged"
   end
 

data/test/test_helper.rb

@@ -27,6 +27,8 @@ class Test::Unit::TestCase
   def stub_request_body(records, time = nil)
     out = ''
     records.each { |record|
+      next unless record.is_a?(Hash)
+
       r = record.dup
       if time
         r['time'] = time

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.23
+  version: 0.10.24
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-08 00:00:00.000000000 Z
+date: 2015-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd