fluent-plugin-td 0.10.3 → 0.10.4

data/ChangeLog

@@ -1,4 +1,10 @@
 
+Release 0.10.4 - 2012/02/09
+
+  * Disable Fluentd's transaction mechanism
+  * Include summary of invalid records in error messages
+
+
 Release 0.10.3 - 2012/02/02
 
   * Support use_ssl parameter

data/VERSION

@@ -1 +1 @@
-0.10.3
+0.10.4

data/lib/fluent/plugin/out_tdlog.rb

@@ -120,10 +120,20 @@ class TreasureDataLogOutput < BufferedOutput
     out = ''
     off = out.bytesize
     es.each {|time,record|
-      record['time'] = time
+      begin
+        record['time'] = time
+
+        if record.size > @key_num_limit
+          raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
+        end
 
-      if record.size > @key_num_limit
-        raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
+      rescue
+        # TODO (a) Remove the transaction mechanism of fluentd
+        #      or (b) keep transaction boundaries in in/out_forward.
+        #      This code disables the transaction mechanism (a).
+        $log.error "#{$!}: #{summarize_record(record)}"
+        $log.error_backtrace $!.backtrace
+        next
       end
 
       record.to_msgpack(out)
@@ -131,13 +141,24 @@ class TreasureDataLogOutput < BufferedOutput
       noff = out.bytesize
       sz = noff - off
       if sz > @record_size_limit
-        raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
+        # TODO don't raise error
+        #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
+        $log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
       end
       off = noff
     }
     out
   end
 
+  def summarize_record(record)
+    json = record.to_json
+    if json.size > 100
+      json[0..97]+"..."
+    else
+      json
+    end
+  end
+
   def write(chunk)
     database, table = chunk.key.split('.',2)
 

data/test/out_tdlog.rb

@@ -50,5 +50,16 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
       d.instance.emit("", es, chain)
     end
   end
+
+  def test_invalid_data
+    d = create_driver
+    d.instance.start
+
+    es = Fluent::OneEventStream.new(Time.now.to_i, "invalid")
+    chain = Fluent::NullOutputChain.instance
+    assert_nothing_raised do
+      d.instance.emit("test.name", es, chain)
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.10.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-02 00:00:00.000000000Z
+date: 2012-02-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &70353352558660 !ruby/object:Gem::Requirement
+  requirement: &70147059008180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.10.6
   type: :runtime
   prerelease: false
-  version_requirements: *70353352558660
+  version_requirements: *70147059008180
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70353352555300 !ruby/object:Gem::Requirement
+  requirement: &70147059007240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.8.7
   type: :runtime
   prerelease: false
-  version_requirements: *70353352555300
+  version_requirements: *70147059007240
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70353352553920 !ruby/object:Gem::Requirement
+  requirement: &70147059006560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70353352553920
+  version_requirements: *70147059006560
 description: Treasure Data output plugin for Fluent event collector
 email: 
 executables: []