fluentd 0.12.15 → 0.12.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e290b6bd8327f7f9fd6a0de891b535de5dbfeb8
-  data.tar.gz: 65429f3882fdcc0bde69322bdc52ff1aaee29e28
+  metadata.gz: 2bc899dd92a360462c8ab40bdec4dfa955918c74
+  data.tar.gz: 0c93a1e695baf87d2d01c0ff10821a2733115c44
 SHA512:
-  metadata.gz: c99e6469d461655c2e43403a6ccb64eb72357853953b8140d1e5ff05a332ada1f3b4a21f74ffd218153cbe68445dc9cb2e0a3972d7a37f3ff9b2f22d26fdbcbb
-  data.tar.gz: f650ce569e09dcf28dc2113a573c88fd35e7f239028159c0f3971e397f32eca694f2ccd0c096084cb90345e0f39b3a4ccb0f48e22ef87a135d490fb7c2e7bc5d
+  metadata.gz: 35585caea38a587df230faa0027cbd5820c6e43e48669d35755055f73a185509e08bf0089c0ff7b2e4c4eaacfe361406f14709161d81f7e3ebc6354c5a203fd1
+  data.tar.gz: 9f3e64fedf79a5609c54e7387a14d2a4c03705b7be00d4382d554a4ff357b5617dcc10689f6e4067ecd8e9e99cf6158583c88ac32761433f04fb091229db6fdc

data/.gitignore

@@ -21,3 +21,4 @@ test/config/tmp/*
 make_dist.sh
 Gemfile.local
 .ruby-version
+coverage/*

data/ChangeLog

@@ -1,5 +1,23 @@
 # v0.12
 
+## Release 0.12.16 - 2015/09/30
+
+### New features / Enhancement
+
+* parser: Add null_value_pattern and null_empty_string option for ltsv, csv and tsv.
+  https://github.com/fluent/fluentd/pull/657
+* RPC: Implement /api/config.dump and /api/config.getDump APIs to dump in memory config
+  https://github.com/fluent/fluentd/pull/666
+* Add --show-plugin-config option
+  https://github.com/fluent/fluentd/pull/663
+
+### Bug fixes
+
+* in_http: Fix add_remote_addr and add_http_headers options not working on bulk request
+  https://github.com/fluent/fluentd/pull/673
+* parser: RegexpParser#initialize should wrap only Hash configuration
+  https://github.com/fluent/fluentd/pull/647
+
 ## Release 0.12.15 - 2015/07/29
 
 ### New features / Enhancement

data/fluentd.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("string-scrub", [">= 0.0.3"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock")
+  gem.add_development_dependency("flexmock", ["~> 1.3.3"])
   gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
   gem.add_development_dependency("simplecov", ["~> 0.6.4"])
   gem.add_development_dependency("rr", [">= 1.0.0"])

data/lib/fluent/command/fluentd.rb

@@ -34,6 +34,10 @@ op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
   opts[:dry_run] = b
 }
 
+op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+  opts[:show_plugin_config] = plugin
+}
+
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
   opts[:plugin_dirs] << s
 }

data/lib/fluent/config/configure_proxy.rb

@@ -17,7 +17,7 @@
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
+      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :descriptions, :sections
       # config_param :desc, :string, :default => '....'
       # config_set_default :buffer_type, :memory
       #
@@ -48,6 +48,7 @@ module Fluent
         @argument = nil # nil: ignore argument
         @params = {}
         @defaults = {}
+        @descriptions = {}
         @sections = {}
       end
 
@@ -157,6 +158,10 @@ module Fluent
           config_set_default(name, opts[:default])
         end
 
+        if opts.has_key?(:desc)
+          config_set_desc(name, opts[:desc])
+        end
+
         [name, block, opts]
       end
 
@@ -189,6 +194,17 @@ module Fluent
         nil
       end
 
+      def config_set_desc(name, description)
+        name = name.to_sym
+
+        if @descriptions.has_key?(name)
+          raise ArgumentError, "#{self.name}: description specified twice for #{name}"
+        end
+
+        @descriptions[name] = description
+        nil
+      end
+
       def config_section(name, *args, &block)
         unless block_given?
           raise ArgumentError, "#{self.name}: config_section requires block parameter"
@@ -208,6 +224,20 @@ module Fluent
 
         name
       end
+
+      def dump(level = 0)
+        dumped_config = "\n"
+        indent = " " * level
+        @params.each do |name, config|
+          dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>"
+          dumped_config << " # #{@descriptions[name]}" if @descriptions[name]
+          dumped_config << "\n"
+        end
+        @sections.each do |section_name, sub_proxy|
+          dumped_config << "#{indent}#{section_name}#{sub_proxy.dump(level + 1)}"
+        end
+        dumped_config
+      end
     end
   end
 end

data/lib/fluent/configurable.rb

@@ -121,6 +121,10 @@ module Fluent
         # p AnyGreatClass.dup.name #=> nil
         configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
       end
+
+      def dump
+        configure_proxy_map[self.to_s].dump
+      end
     end
   end
 

data/lib/fluent/output.rb

@@ -527,7 +527,7 @@ module Fluent
       else
         @flush_interval = [60, @time_slice_cache_interval].min
         @enqueue_buffer_proc = Proc.new do
-          nowslice = @time_slicer.call(Engine.now.to_i - @time_slice_wait)
+          nowslice = @time_slicer.call(Engine.now - @time_slice_wait)
           @buffer.keys.each {|key|
             if key < nowslice
               @buffer.push(key)

data/lib/fluent/parser.rb

@@ -169,7 +169,7 @@ module Fluent
         super()
         @regexp = regexp
         unless conf.empty?
-          conf = Config::Element.new('default_regexp_conf', '', conf, [])
+          conf = Config::Element.new('default_regexp_conf', '', conf, []) unless conf.is_a?(Config::Element)
           configure(conf)
         end
 
@@ -295,6 +295,8 @@ module Fluent
       end
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
+      config_param :null_value_pattern, :string, :default => nil
+      config_param :null_empty_string, :bool, :default => false
 
       def configure(conf)
         super
@@ -308,11 +310,16 @@ module Fluent
         end
 
         @time_parser = TimeParser.new(@time_format)
+
+        if @null_value_pattern
+          @null_value_pattern = Regexp.new(@null_value_pattern)
+        end
+
         @mutex = Mutex.new
       end
 
       def values_map(values)
-        record = Hash[keys.zip(values)]
+        record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
 
         if @time_key
           value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
@@ -345,6 +352,16 @@ module Fluent
           end
         }
       end
+
+      def convert_value_to_nil(value)
+        if value and @null_empty_string
+          value = (value == '') ? nil : value
+        end
+        if value and @null_value_pattern
+          value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+        end
+        value
+      end
     end
 
     class TSVParser < ValuesParser

data/lib/fluent/plugin/filter_grep.rb

@@ -18,6 +18,11 @@ module Fluent
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
 
+    def initialize
+      super
+      require 'fluent/plugin/string_util'
+    end
+
     REGEXP_MAX_NUM = 20
 
     (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
@@ -54,10 +59,10 @@ module Fluent
       begin
         catch(:break_loop) do
           @regexps.each do |key, regexp|
-            throw :break_loop unless match(regexp, record[key].to_s)
+            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
           end
           @excludes.each do |key, exclude|
-            throw :break_loop if match(exclude, record[key].to_s)
+            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
           end
           result = record
         end
@@ -67,19 +72,5 @@ module Fluent
       end
       result
     end
-
-    private
-
-    def match(regexp, string)
-      begin
-        return regexp.match(string)
-      rescue ArgumentError => e
-        raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-        log.info "invalid byte sequence is replaced in `#{string}`"
-        string = string.scrub('?')
-        retry
-      end
-      return true
-    end
   end
 end

data/lib/fluent/plugin/in_http.rb

@@ -131,18 +131,18 @@ module Fluent
           end
         end
 
-        if @add_http_headers
-          params.each_pair { |k,v|
-            if k.start_with?("HTTP_")
-              record[k] = v
-            end
-          }
-        end
-
-        if @add_remote_addr
-          record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+        unless record.is_a?(Array)
+          if @add_http_headers
+            params.each_pair { |k,v|
+              if k.start_with?("HTTP_")
+                record[k] = v
+              end
+            }
+          end
+          if @add_remote_addr
+            record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+          end
         end
-
         time = if param_time = params['time']
                  param_time = param_time.to_i
                  param_time.zero? ? Engine.now : param_time
@@ -159,11 +159,21 @@ module Fluent
         if record.is_a?(Array)
           mes = MultiEventStream.new
           record.each do |single_record|
+            if @add_http_headers
+              params.each_pair { |k,v|
+                if k.start_with?("HTTP_")
+                  single_record[k] = v
+                end
+              }
+            end
+            if @add_remote_addr
+              single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+            end
             single_time = single_record.delete("time") || time
             mes.add(single_time, single_record)
           end
           router.emit_stream(tag, mes)
-	else
+        else
           router.emit(tag, time, record)
         end
       rescue

data/lib/fluent/plugin/string_util.rb

@@ -0,0 +1,32 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module StringUtil
+    def match_regexp(regexp, string)
+      begin
+        return regexp.match(string)
+      rescue ArgumentError => e
+        raise e unless e.message.index("invalid byte sequence in".freeze).zero?
+        log.info "invalid byte sequence is replaced in `#{string}`"
+        string = string.scrub('?')
+        retry
+      end
+      return true
+    end
+    module_function :match_regexp
+  end
+end

data/lib/fluent/rpc.rb

@@ -41,7 +41,7 @@ module Fluent
       def mount_proc(path, &block)
         @server.mount_proc(path) { |req, res|
           begin
-            code, header, body = block.call(req, res)
+            code, header, response = block.call(req, res)
           rescue => e
             @log.warn "failed to handle RPC request", :path => path, :error => e.to_s
             @log.warn_backtrace e.backtrace
@@ -56,11 +56,11 @@ module Fluent
 
           code = 200 if code.nil?
           header = {'Content-Type' => 'application/json'} if header.nil?
-          body = if body.nil?
+          body = if response.nil?
                    '{"ok":true}'
                  else
-                   body['ok'] = code == 200
-                   body.to_json
+                   response.body['ok'] = code == 200
+                   response.body.to_json
                  end
 
           res.status = code

data/lib/fluent/supervisor.rb

@@ -105,6 +105,7 @@ module Fluent
       @use_v1_config = opt[:use_v1_config]
       @log_path = opt[:log_path]
       @dry_run = opt[:dry_run]
+      @show_plugin_config = opt[:show_plugin_config]
       @libs = opt[:libs]
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
@@ -124,12 +125,15 @@ module Fluent
 
     def start
       @log.init
+      show_plugin_config if @show_plugin_config
       read_config
       apply_system_config
 
       dry_run if @dry_run
       start_daemonize if @daemonize
       setup_rpc_server if @rpc_endpoint
+      setup_rpc_get_dump if @enable_get_dump
+
       if @supervise
         install_supervisor_signal_handlers
         run_rpc_server if @rpc_endpoint
@@ -185,6 +189,17 @@ module Fluent
       exit 1
     end
 
+    def show_plugin_config
+      $log.info "Show config for #{@show_plugin_config}"
+      name, type = @show_plugin_config.split(":")
+      plugin = Plugin.__send__("new_#{name}", type)
+      $log.info plugin.class.dump
+      exit 0
+    rescue => e
+      $log.error "show config failed: #{e}"
+      exit 1
+    end
+
     def start_daemonize
       @wait_daemonize_pipe_r, @wait_daemonize_pipe_w = IO.pipe
 
@@ -263,6 +278,21 @@ module Fluent
         supervisor_sighup_handler
         nil
       }
+      @rpc_server.mount_proc('/api/config.dump') { |req, res|
+        $log.debug "fluentd RPC got /api/config.dump request"
+        $log.info "dump in-memory config"
+        supervisor_dump_config_handler
+        nil
+      }
+    end
+
+    def setup_rpc_get_dump
+      @rpc_server.mount_proc('/api/config.getDump') { |req, res|
+        $log.debug "fluentd RPC got /api/config.dump request"
+        $log.info "get dump in-memory config via HTTP"
+        res.body = supervisor_get_dump_config_handler
+        [nil, nil, res]
+      }
     end
 
     def run_rpc_server
@@ -399,6 +429,14 @@ module Fluent
       end
     end
 
+    def supervisor_dump_config_handler
+      $log.info @conf.to_s
+    end
+
+    def supervisor_get_dump_config_handler
+      {conf: @conf.to_s}
+    end
+
     def read_config
       $log.info "reading config file", :path => @config_path
       @config_fname = File.basename(@config_path)
@@ -423,6 +461,7 @@ module Fluent
       config_param :suppress_config_dump, :bool, :default => nil
       config_param :without_source, :bool, :default => nil
       config_param :rpc_endpoint, :string, :default => nil
+      config_param :enable_get_dump, :bool, :default => nil
 
       def initialize(conf)
         super()
@@ -438,6 +477,7 @@ module Fluent
           @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
           @without_source = system.without_source unless system.without_source.nil?
           @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
+          @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
         }
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.15'
+  VERSION = '0.12.16'
 
 end

data/test/config/test_configure_proxy.rb

@@ -94,6 +94,128 @@ module Fluent::Config
           assert_raise(ArgumentError) { proxy.config_argument(:name, default: "name2") }
         end
       end
+
+      sub_test_case '#config_set_desc' do
+        setup do
+          @proxy = Fluent::Config::ConfigureProxy.new(:section)
+        end
+
+        test 'does not permit description specification twice w/ :desc option' do
+          @proxy.config_param(:name, :string, desc: "description")
+          assert_raise(ArgumentError) { @proxy.config_set_desc(:name, "description2") }
+        end
+
+        test 'does not permit description specification twice' do
+          @proxy.config_param(:name, :string)
+          @proxy.config_set_desc(:name, "description")
+          assert_raise(ArgumentError) { @proxy.config_set_desc(:name, "description2") }
+        end
+      end
+
+      sub_test_case '#dump' do
+        setup do
+          @proxy = Fluent::Config::ConfigureProxy.new(:section)
+        end
+
+        test 'empty proxy' do
+          assert_equal("\n", @proxy.dump)
+        end
+
+        test 'plain proxy w/o default value' do
+          @proxy.config_param(:name, :string)
+          assert_equal(<<CONFIG, @proxy.dump)
+
+name: string: <nil>
+CONFIG
+        end
+
+        test 'plain proxy w/ default value' do
+          @proxy.config_param(:name, :string, default: "name1")
+          assert_equal(<<CONFIG, @proxy.dump)
+
+name: string: <"name1">
+CONFIG
+        end
+
+        test 'plain proxy w/ default value using config_set_default' do
+          @proxy.config_param(:name, :string)
+          @proxy.config_set_default(:name, "name1")
+          assert_equal(<<CONFIG, @proxy.dump)
+
+name: string: <"name1">
+CONFIG
+        end
+
+        test 'single sub proxy' do
+          @proxy.config_section(:sub) do
+            config_param(:name, :string, default: "name1")
+          end
+          assert_equal(<<CONFIG, @proxy.dump)
+
+sub
+ name: string: <"name1">
+CONFIG
+        end
+
+        test 'nested sub proxy' do
+          @proxy.config_section(:sub) do
+            config_param(:name1, :string, default: "name1")
+            config_param(:name2, :string, default: "name2")
+            config_section(:sub2) do
+              config_param(:name3, :string, default: "name3")
+              config_param(:name4, :string, default: "name4")
+            end
+          end
+          assert_equal(<<CONFIG, @proxy.dump)
+
+sub
+ name1: string: <"name1">
+ name2: string: <"name2">
+ sub2
+  name3: string: <"name3">
+  name4: string: <"name4">
+CONFIG
+        end
+
+        sub_test_case 'w/ description' do
+          test 'single proxy' do
+            @proxy.config_param(:name, :string, desc: "description for name")
+          assert_equal(<<CONFIG, @proxy.dump)
+
+name: string: <nil> # description for name
+CONFIG
+          end
+
+          test 'single proxy using config_set_desc' do
+            @proxy.config_param(:name, :string)
+            @proxy.config_set_desc(:name, "description for name")
+          assert_equal(<<CONFIG, @proxy.dump)
+
+name: string: <nil> # description for name
+CONFIG
+          end
+
+          test 'sub proxy' do
+            @proxy.config_section(:sub) do
+              config_param(:name1, :string, default: "name1", desc: "desc1")
+              config_param(:name2, :string, default: "name2", desc: "desc2")
+              config_section(:sub2) do
+                config_param(:name3, :string, default: "name3")
+                config_param(:name4, :string, default: "name4", desc: "desc4")
+              end
+            end
+            assert_equal(<<CONFIG, @proxy.dump)
+
+sub
+ name1: string: <"name1"> # desc1
+ name2: string: <"name2"> # desc2
+ sub2
+  name3: string: <"name3">
+  name4: string: <"name4"> # desc4
+CONFIG
+          end
+        end
+      end
     end
   end
 end

data/test/plugin/test_in_http.rb

@@ -85,6 +85,56 @@ class HttpInputTest < Test::Unit::TestCase
 
   end
 
+  def test_json_with_add_remote_addr
+    d = create_driver(CONFIG + "add_remote_addr true")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>1}
+    d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time.to_s})
+        assert_equal "200", res.code
+      }
+    end
+
+  end
+
+  def test_multi_json_with_add_remote_addr
+    d = create_driver(CONFIG + "add_remote_addr true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>1}
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"127.0.0.1", "a"=>2}
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
+      assert_equal "200", res.code
+    end
+
+  end
+
+  def test_multi_json_with_add_http_headers
+    d = create_driver(CONFIG + "add_http_headers true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
+      assert_equal "200", res.code
+    end
+
+    d.emit_streams.each { |tag, es|
+      assert include_http_header?(es.first[1])
+    }
+  end
+
   def test_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
 

data/test/test_output.rb

@@ -213,7 +213,7 @@ module FluentOutputTest
       FileUtils.mkdir_p(TMP_DIR)
     end
 
-    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/time_sliced_output")
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/time_sliced_output")
 
     CONFIG = %[]
 

data/test/test_parser.rb

@@ -529,6 +529,38 @@ module ParserTest
         assert_equal text, record['time']
       end
     end
+
+    data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
+    def test_parse_with_null_value_pattern
+      parser = TextParser::TSVParser.new
+      parser.configure(
+        'keys'=>param,
+        'time_key'=>'time',
+        'null_value_pattern'=>'^(-|null|NULL)$'
+      )
+      parser.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
+        assert_nil record['a']
+        assert_nil record['b']
+        assert_nil record['c']
+        assert_equal record['d'], ''
+        assert_equal record['e'], '--'
+        assert_equal record['f'], 'nuLL'
+      end
+    end
+
+    data('array param' => '["a","b"]', 'string param' => 'a,b')
+    def test_parse_with_null_empty_string
+      parser = TextParser::TSVParser.new
+      parser.configure(
+        'keys'=>param,
+        'time_key'=>'time',
+        'null_empty_string'=>true
+      )
+      parser.parse("\t ") do |time, record|
+        assert_nil record['a']
+        assert_equal record['b'], ' '
+      end
+    end
   end
 
   class CSVParserTest < ::Test::Unit::TestCase
@@ -586,6 +618,38 @@ module ParserTest
         assert_equal text, record['time']
       end
     end
+
+    data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
+    def test_parse_with_null_value_pattern
+      parser = TextParser::CSVParser.new
+      parser.configure(
+        'keys'=>param,
+        'time_key'=>'time',
+        'null_value_pattern'=>'^(-|null|NULL)$'
+      )
+      parser.parse("-,null,NULL,,--,nuLL") do |time, record|
+        assert_nil record['a']
+        assert_nil record['b']
+        assert_nil record['c']
+        assert_equal record['d'], ''
+        assert_equal record['e'], '--'
+        assert_equal record['f'], 'nuLL'
+      end
+    end
+
+    data('array param' => '["a","b"]', 'string param' => 'a,b')
+    def test_parse_with_null_empty_string
+      parser = TextParser::CSVParser.new
+      parser.configure(
+        'keys'=>param,
+        'time_key'=>'time',
+        'null_empty_string'=>true
+      )
+      parser.parse(", ") do |time, record|
+        assert_nil record['a']
+        assert_equal record['b'], ' '
+      end
+    end
   end
 
   class LabeledTSVParserTest < ::Test::Unit::TestCase
@@ -684,6 +748,32 @@ module ParserTest
         assert_equal text, record['time']
       end
     end
+
+    def test_parse_with_null_value_pattern
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure(
+        'null_value_pattern'=>'^(-|null|NULL)$'
+      )
+      parser.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
+        assert_nil record['a']
+        assert_nil record['b']
+        assert_nil record['c']
+        assert_equal record['d'], ''
+        assert_equal record['e'], '--'
+        assert_equal record['f'], 'nuLL'
+      end
+    end
+
+    def test_parse_with_null_empty_string
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure(
+        'null_empty_string'=>true
+      )
+      parser.parse("a:\tb: ") do |time, record|
+        assert_nil record['a']
+        assert_equal record['b'], ' '
+      end
+    end
   end
 
   class NoneParserTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.15
+  version: 0.12.16
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-29 00:00:00.000000000 Z
+date: 2015-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -172,16 +172,16 @@ dependencies:
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
@@ -370,6 +370,7 @@ files:
 - lib/fluent/plugin/out_stdout.rb
 - lib/fluent/plugin/out_stream.rb
 - lib/fluent/plugin/socket_util.rb
+- lib/fluent/plugin/string_util.rb
 - lib/fluent/process.rb
 - lib/fluent/registry.rb
 - lib/fluent/root_agent.rb