fluent-mixin-plaintextformatter 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3f892bb7a90ccd8e91cae4e9581ac540b1365c4
-  data.tar.gz: 04b6d2744aa25081c5e8e6b7f629f0b4d60fa9de
+  metadata.gz: 8a8f706410fb735c2cf645b5a11eb1b67f6afc82
+  data.tar.gz: dc8832c1ab304abe876300f83a367fd7201ed216
 SHA512:
-  metadata.gz: 6a5b50af5b252f1b30c292cab814b501ca84656f3dc802cb5eed660973ade3bf1351e44ddc457071b004470c21b5f0e0823be0e4cdb40d91a0625c9908fe3b19
-  data.tar.gz: e6c183f7a11465843e1aa4fc291f572ee03ef839bcae67e2240c011f080e989fc67b296261365289efa894ef121610b92b0e036bb293510e646e0aaff0d1096b
+  metadata.gz: 8b18fcd7738dd940b2d297e9b5e8802e20653bc4eaa189801af644c39361b1aa32bcfc430ddd93746a7c2a0cb0afbcff6bd9b3f1c2978fb384ee7295351c6df2
+  data.tar.gz: 9ee257ee3a95d62953e8e343139bc4634d9c91e6ab0ccc0b84f28a9c53568d5f54c28f4f129efb2515f35e45e9221b823b0e985d4f492ef0f24ec75c0ac4657a

data/fluent-mixin-plaintextformatter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-mixin-plaintextformatter"
-  gem.version       = "0.2.3"
+  gem.version       = "0.2.4"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{included to format values into json, tsv or csv}

data/lib/fluent/mixin/plaintextformatter.rb

@@ -8,6 +8,8 @@ module Fluent
       attr_accessor :add_newline, :field_separator
       attr_accessor :remove_prefix, :default_tag
 
+      attr_accessor :suppress_log_broken_string
+
       attr_accessor :f_separator
 
       def first_value(*args)
@@ -65,6 +67,12 @@ module Fluent
                              else
                                raise Fluent::ConfigError, "invalid output_data_type:'#{@output_data_type}'"
                              end
+
+        @suppress_log_broken_string = first_value(
+          Fluent::Config.bool_value(conf['suppress_log_broken_string']),
+          @suppress_log_broken_string,
+          false
+        )
       end
 
       def stringify_record(record)
@@ -105,11 +113,15 @@ module Fluent
         rescue JSON::GeneratorError => e
           # partial character in source, but hit end
           # source sequence is illegal/malformed utf-8
-          $log.error e.message + ", ignored", :error_class => e.class, :tag => tag, :record => record.inspect # quote explicitly
+          unless @suppress_log_broken_string
+            $log.warn e.message + ", ignored", :error_class => e.class, :tag => tag, :record => record.inspect # quote explicitly
+          end
           ''
         rescue ArgumentError => e
           raise unless e.message == 'invalid byte sequence in UTF-8'
-          $log.error e.message + ", ignored", :error_class => e.class, :tag => tag, :record => record.inspect # quote explicitly
+          unless @suppress_log_broken_string
+            $log.warn e.message + ", ignored", :error_class => e.class, :tag => tag, :record => record.inspect # quote explicitly
+          end
           ''
         end
       end

data/test/helper.rb

@@ -16,8 +16,11 @@ require 'fluent/test'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|
+    def logs ; @logs ; end
+    def clear ; @logs = [] ; end
     def method_missing(method, *args)
-      # pass
+      @logs ||= []
+      @logs.push([method, *args])
     end
   }
   $log = nulllogger

data/test/mixin/test_plaintextformatter.rb

@@ -175,6 +175,47 @@ field_separator comma
   end
 
   def test_format_invalid_utf8_sequence
+    $log.clear
+
+    invalid_str = [0xFA, 0xFB].pack('CC').force_encoding('utf-8')
+    valid_str = [0xFF, 0xE3].pack("U*")
+
+    p1 = create_plugin_instance(Fluent::TestAOutput, %[
+type testa
+output_include_time true
+output_include_tag  true
+output_data_type json
+])
+    r1 = p1.format('tag', Fluent::Engine.now, {'foo' => valid_str, 'bar' => invalid_str + valid_str})
+    # #format should logs for this record (but we cannot test it...)
+    assert_equal '', r1
+
+    assert_equal :warn, $log.logs[0][0]
+    assert       $log.logs[0][1] =~ /source sequence is illegal\/malformed utf-8, ignored/
+    assert_equal 'JSON::GeneratorError', $log.logs[0][2][:error_class].to_s
+
+    $log.clear
+
+    p2 = create_plugin_instance(Fluent::TestAOutput, %[
+type testa
+output_include_time true
+output_include_tag  true
+output_data_type ltsv
+])
+    r2 = p2.format('tag', Fluent::Engine.now, {'foo' => valid_str, 'bar' => invalid_str + valid_str})
+    # #format should logs for this record (but we cannot test it...)
+    assert_equal '', r2
+
+    assert_equal :warn, $log.logs[0][0]
+    assert       $log.logs[0][1] =~ /^invalid byte sequence in UTF-8, ignored/
+    assert_equal 'ArgumentError', $log.logs[0][2][:error_class].to_s
+
+    $log.clear
+  end
+
+  def test_format_invalid_utf8_sequence_suppress_logs
+    $log.clear
+
     invalid_str = [0xFA, 0xFB].pack('CC').force_encoding('utf-8')
     valid_str = [0xFF, 0xE3].pack("U*")
 
@@ -183,20 +224,30 @@ type testa
 output_include_time true
 output_include_tag  true
 output_data_type json
+suppress_log_broken_string true
 ])
     r1 = p1.format('tag', Fluent::Engine.now, {'foo' => valid_str, 'bar' => invalid_str + valid_str})
     # #format should logs for this record (but we cannot test it...)
     assert_equal '', r1
 
+    assert_equal 0, $log.logs.size
+
+    $log.clear
+
     p2 = create_plugin_instance(Fluent::TestAOutput, %[
 type testa
 output_include_time true
 output_include_tag  true
 output_data_type ltsv
+suppress_log_broken_string true
 ])
     r2 = p2.format('tag', Fluent::Engine.now, {'foo' => valid_str, 'bar' => invalid_str + valid_str})
     # #format should logs for this record (but we cannot test it...)
     assert_equal '', r2
+
+    assert_equal 0, $log.logs.size
+
+    $log.clear
   end
 
   def test_field_separator_newline_ltsv

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-mixin-plaintextformatter
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-30 00:00:00.000000000 Z
+date: 2013-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd