fluentd 0.12.32 → 0.12.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29eef51ecd637ecb6cf5cc7d41bc31527513afff
-  data.tar.gz: 2077e5585e911081e2f76225ff97995aaa1b5100
+  metadata.gz: daedf4abbe7d43acdd8df2b8ac75f4f93dc8f252
+  data.tar.gz: 99ea120dbf1257a4be97fc480a4f022b7cc638bf
 SHA512:
-  metadata.gz: 416d42c9f2d15131fbe0b01c91a11b5cb22f3e54d052ebc948304e4f9a02ca1445f5c88882e9af01778e553fcb708c83b24a42b3442582a9e1ee8e6ad1ebf9e0
-  data.tar.gz: fee4ebcca6c1ee8eb452043552bc720fb8850641414bbf4586d9d668a93a9230c1ed6283220255d4150b7ae135af4010590082af962871bfae4ca8d0c37c2325
+  metadata.gz: d9440abbc6e21242e9281fe3c9ca469fa12f9b47abe8115559c4702fdb5f763fa8576d986434ea76829da45b21a05edfead3217d8ab5cfc798f28de2dea0d982
+  data.tar.gz: dab81f6a5f102c884c0ab0ec6628934aeb5fff690e87ea24148b421fae70e8efab72425b59e8f558052fb8e7f6dd3a097a1268b9af794abd8e4f5e0397b67176

data/ChangeLog

@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.33 - 2017/03/09
+
+### New features / Enhancement
+
+* in_syslog: Backport 'Support rfc5424 syslog format'
+  https://github.com/fluent/fluentd/pull/1495
+* in_forward: Add source_address_key and fix source_hostname_key parameters
+  https://github.com/fluent/fluentd/pull/1490
+* in_tail: Skip directories when use **/* in path
+  https://github.com/fluent/fluentd/pull/1464
+* in_tail: Add limit_recently_modified parameter to limit watching files
+  https://github.com/fluent/fluentd/pull/1474
+* in_tail: Skip the refresh of watching list on startup
+  https://github.com/fluent/fluentd/pull/1487
+* parser: Allow escape sequence in Apache access log
+  https://github.com/fluent/fluentd/pull/1479
+* log: Add Fluent::Log#<< to support some SDKs
+  https://github.com/fluent/fluentd/pull/1478
+
+### Bug fixes
+
+* config: Set encoding forcefully to avoid UndefinedConversionError
+  https://github.com/fluent/fluentd/pull/1477
+
 ## Release 0.12.32 - 2017/02/02
 
 ### New features / Enhancement

data/README.md

@@ -26,6 +26,31 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
     $ fluentd -c conf/fluent.conf &
     $ echo '{"json":"message"}' | fluent-cat debug.test
 
+## Development
+
+### Prerequisites
+
+- Ruby 2.1 or later
+- git
+
+`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
+
+### Install dependent gems
+
+Use bundler:
+
+    $ gem install bundler
+    $ bundle install --path vendor/bundle
+
+### Run test
+
+    $ bundle exec rake test
+
+You can run specified test via `TEST` environment variable:
+
+    $ bundle exec rake test TEST=test/test_specified_path.rb
+    $ bundle exec rake test TEST=test/test_*.rb
+
 ## Fluentd UI: Admin GUI
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.

data/lib/fluent/config/types.rb

@@ -65,7 +65,7 @@ module Fluent
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val }
+    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -86,7 +86,7 @@ module Fluent
         value
       else
         case type
-        when :string  then value.to_s
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
         when :integer then value.to_i
         when :float   then value.to_f
         when :size then Config.size_value(value)

data/lib/fluent/log.rb

@@ -240,6 +240,9 @@ module Fluent
     def write(data)
       @out.write(data)
     end
+    # We need `#<<` method to use this logger class with other
+    # libraries such as aws-sdk
+    alias << write
 
     def flush
       @out.flush

data/lib/fluent/parser.rb

@@ -463,7 +463,7 @@ module Fluent
     end
 
     class ApacheParser < Parser
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
@@ -536,9 +536,13 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_param :time_format, :string, default: "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
         super
@@ -548,7 +552,27 @@ module Fluent
       def configure(conf)
         super
 
-        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
+        @regexp = case @message_format
+                  when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
+                    REGEXP_RFC5424
+                  when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
+                    @time_parser_rfc3164 = TextParser::TimeParser.new(@time_format)
+                    @time_parser_rfc5424 = TextParser::TimeParser.new(@rfc5424_time_format)
+                    nil
+                  end
         @time_parser = TextParser::TimeParser.new(@time_format)
       end
 
@@ -557,6 +581,21 @@ module Fluent
       end
 
       def parse(text)
+        # This is overwritten in configure
+      end
+
+      def parse_auto(text, &block)
+        if REGEXP_DETECT_RFC5424.match(text)
+          @regexp = REGEXP_RFC5424
+          @time_parser = @time_parser_rfc5424
+        else
+          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @time_parser = @time_parser_rfc3164
+        end
+        parse_plain(text, &block)
+      end
+
+      def parse_plain(text, &block)
         m = @regexp.match(text)
         unless m
           if block_given?

data/lib/fluent/plugin/in_forward.rb

@@ -47,11 +47,24 @@ module Fluent
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+    desc 'Try to resolve hostname from IP addresses or not.'
+    config_param :resolve_hostname, :bool, default: nil
+    desc "The field name of the client's source address."
+    config_param :source_address_key, :string, default: nil
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
+
+      if @source_hostname_key
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in configure
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
+      @enable_field_injection = @source_address_key || @source_hostname_key
     end
 
     def start
@@ -88,7 +101,7 @@ module Fluent
 
     def listen
       log.info "listening fluent socket on #{@bind}:#{@port}"
-      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, method(:on_message))
+      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, @resolve_hostname, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s
     end
@@ -161,7 +174,7 @@ module Fluent
         # PackedForward
         es = MessagePackEventStream.new(entries)
         es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = add_source_host(es, peeraddr) if @enable_field_injection
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -180,7 +193,7 @@ module Fluent
                }
                es
              end
-        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
+        es = add_source_host(es, peeraddr) if @enable_field_injection
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -194,7 +207,10 @@ module Fluent
         end
         return if record.nil?
         time = Engine.now if time == 0
-        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
+        if @enable_field_injection
+          record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
+          record[@source_address_key] = peeraddr[3] if @source_address_key
+        end
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -219,12 +235,31 @@ module Fluent
       new_es
     end
 
-    def add_source_host(es, host)
+    def add_source_host(es, peeraddr)
       new_es = MultiEventStream.new
-      es.each { |time, record|
-        record[@source_hostname_key] = host
-        new_es.add(time, record)
-      }
+      if @source_address_key && @source_hostname_key
+        address = peeraddr[3]
+        hostname = peeraddr[2]
+        es.each { |time, record|
+          record[@source_address_key] = address
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      elsif @source_address_key
+        address = peeraddr[3]
+        es.each { |time, record|
+          record[@source_address_key] = address
+          new_es.add(time, record)
+        }
+      elsif @source_hostname_key
+        hostname = peeraddr[2]
+        es.each { |time, record|
+          record[@source_hostname_key] = hostname
+          new_es.add(time, record)
+        }
+      else
+        raise "BUG: don't call this method in this case"
+      end
       new_es
     end
 
@@ -236,11 +271,13 @@ module Fluent
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
-      def initialize(io, linger_timeout, log, on_message)
+      def initialize(io, linger_timeout, log, resolve_hostname, on_message)
         super(io)
 
         @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
+          io.do_not_reverse_lookup = !resolve_hostname unless resolve_hostname.nil?
+
           @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)

data/lib/fluent/plugin/in_tail.rb

@@ -58,6 +58,10 @@ module Fluent
     config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
+    desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
+    config_param :limit_recently_modified, :time, default: nil
+    desc 'Enable the option to skip the refresh of watching list on startup.'
+    config_param :skip_refresh_on_startup, :bool, default: false
 
     attr_reader :paths
 
@@ -128,7 +132,7 @@ module Fluent
       end
 
       @loop = Coolio::Loop.new
-      refresh_watchers
+      refresh_watchers unless @skip_refresh_on_startup
 
       @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
       @refresh_trigger.attach(@loop)
@@ -153,8 +157,12 @@ module Fluent
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            if File.readable?(p)
-              true
+            if File.readable?(p) && !File.directory?(p)
+              if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                false
+              else
+                true
+              end
             else
               log.warn "#{p} unreadable. It is excluded and would be examined next time."
               false

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.32'
+  VERSION = '0.12.33'
 
 end

data/test/config/test_types.rb

@@ -68,6 +68,17 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal '   ', Config::STRING_TYPE.call('   ', {})
     end
 
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
     test 'enum' do
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
@@ -142,6 +153,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
       assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))

data/test/plugin/test_in_forward.rb

@@ -498,52 +498,104 @@ class ForwardInputTest < Test::Unit::TestCase
     @responses << res if try_to_receive_response
   end
 
-  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
-  test 'message protocol with source_hostname_key' do
-    execute_test { |events|
-      events.each { |tag, time, record|
-        send_data [tag, time, record].to_msgpack
+  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
+  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
+  DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
+
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    test 'resolve_hostname must be true with source_hostname_key' do
+      assert_raise(Fluent::ConfigError) {
+        create_driver(CONFIG + <<EOS)
+resolve_hostname false
+source_hostname_key hostname
+EOS
       }
-    }
-  end
+    end
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'message protocol' do |keys|
+      execute_test(*keys) { |events|
+        events.each { |tag, time, record|
+          send_data [tag, time, record].to_msgpack
+        }
+      }
+    end
 
-  test 'forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = []
-      events.each {|tag,time,record|
-        entries << [time, record]
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'forward protocol' do |keys|
+      execute_test(*keys) { |events|
+        entries = []
+        events.each {|tag,time,record|
+          entries << [time, record]
+        }
+        send_data ['tag1', entries].to_msgpack
       }
-      send_data ['tag1', entries].to_msgpack
-    }
-  end
+    end
 
-  test 'packed forward protocol with source_hostname_key' do
-    execute_test { |events|
-      entries = ''
-      events.each { |tag, time, record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'packed forward protocol' do |keys|
+      execute_test(*keys) { |events|
+        entries = ''
+        events.each { |tag, time, record|
+          Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+        }
+        send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
       }
-      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
-    }
-  end
+    end
 
-  def execute_test(&block)
-    d = create_driver(CONFIG + 'source_hostname_key source')
+    def execute_test(*keys, &block)
+      conf = CONFIG.dup
+      if keys.include?(:hostname)
+        conf << <<EOL
+source_hostname_key source_hostname
+EOL
+      end
+      if keys.include?(:address)
+        conf << <<EOL
+source_address_key source_address
+EOL
+      end
+      d = create_driver(conf)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    events = [
-      ["tag1", time, {"a"=>1}],
-      ["tag1", time, {"a"=>2}]
-    ]
-    d.expected_emits_length = events.length
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      events = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}]
+      ]
+      d.expected_emits_length = events.length
 
-    d.run do
-      block.call(events)
-    end
+      d.run do
+        block.call(events)
+      end
 
-    d.emits.each { |tag, _time, record|
-      assert_true record.has_key?('source')
-    }
+      d.emits.each { |tag, _time, record|
+        if keys.include?(:hostname)
+          assert_true record.has_key?('source_hostname')
+          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
+          unless keys.include?(:address)
+            assert_false record.has_key?('source_address')
+          end
+        end
+        if keys.include?(:address)
+          assert_true record.has_key?('source_address')
+          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
+          unless keys.include?(:hostname)
+            assert_false record.has_key?('source_hostname')
+          end
+        end
+      }
+    end
   end
 
   # TODO heartbeat

data/test/plugin/test_in_tail.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/test'
 require 'net/http'
 require 'flexmock/test_unit'
+require 'timecop'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -697,6 +698,25 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
   end
 
+  def test_log_file_without_extension
+    expected_files = [
+      'test/plugin/data/log/bar',
+      'test/plugin/data/log/foo/bar.log',
+      'test/plugin/data/log/foo/bar2',
+      'test/plugin/data/log/test.log'
+    ]
+
+    config = config_element("", "", {
+      "tag" => "tail",
+      "path" => "test/plugin/data/log/**/*",
+      "format" => "none",
+      "pos_file" => "#{TMP_DIR}/tail.pos"
+    })
+
+    plugin = create_driver(config, false).instance
+    assert_equal expected_files, plugin.expand_paths.sort
+  end
+
   def test_refresh_watchers
     plugin = create_driver(EX_CONFIG, false).instance
     sio = StringIO.new
@@ -992,4 +1012,43 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
     end
   end
+
+  def test_limit_recently_modified
+    now = Time.new(2010, 1, 2, 3, 4, 5)
+    FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
+    FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
+    FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
+
+    config = config_element('', '', {
+      'tag' => 'tail',
+      'path' => "#{TMP_DIR}/*.txt",
+      'format' => 'none',
+      'limit_recently_modified' => '3600s'
+    })
+
+    expected_files = [
+      "#{TMP_DIR}/tail_watch1.txt",
+      "#{TMP_DIR}/tail_watch2.txt"
+    ]
+
+    Timecop.freeze(now) do
+      plugin = create_driver(config, false).instance
+      assert_equal expected_files, plugin.expand_paths.sort
+    end
+  end
+
+  def test_skip_refresh_on_startup
+    FileUtils.touch("#{TMP_DIR}/tail.txt")
+    config = config_element('', '', {
+                              'tag' => 'tail',
+                              'path' => "#{TMP_DIR}/*.txt",
+                              'format' => 'none',
+                              'refresh_interval' => 1,
+                              'skip_refresh_on_startup' => true
+                            })
+    d = create_driver(config, false)
+    d.run {
+      sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1
+    }
+  end
 end

data/test/test_log.rb

@@ -305,6 +305,10 @@ class PluginLoggerTest < Test::Unit::TestCase
       @log.write("log")
     end
 
+    def test_write_alias
+      assert(@log.respond_to?(:<<))
+    end
+
     def test_out
       assert_equal(@log.out, @logger.out)
       @log.out = Object.new

data/test/test_parser.rb

@@ -292,6 +292,14 @@ module ParserTest
         assert_equal(@expected, record)
       }
     end
+
+    def test_parse_with_escape_sequence
+      @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
+        assert_equal('/\"', record['path'])
+        assert_equal('referer \\\ \"', record['referer'])
+        assert_equal('user agent \\\ \"', record['agent'])
+      }
+    end
   end
 
   class SyslogParserTest < ::Test::Unit::TestCase
@@ -356,6 +364,182 @@ module ParserTest
         assert_equal "Feb 28 00:00:12", record['time']
       end
     end
+
+    class TestRFC5424Regexp < self
+      def test_parse_with_rfc5424_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_rfc5424_message_without_time_format
+        @parser.configure(
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.instance.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_rfc5424_structured_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'rfc5424',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+    end
+
+    class TestAutoRegexp < self
+      def test_auto_with_legacy_syslog_message
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'mseeage_format' => 'auto',
+        )
+        text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+      end
+
+      def test_auto_with_legacy_syslog_priority_message
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'with_priority' => true,
+          'mseeage_format' => 'auto',
+        )
+        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 6), record)
+        end
+      end
+
+      def test_parse_with_rfc5424_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_rfc5424_structured_message
+        @parser.configure(
+          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_both_message_type
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'message_format' => 'auto',
+        )
+        text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+        text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected, record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+
+      def test_parse_with_both_message_type_and_priority
+        @parser.configure(
+          'time_format' => '%b %d %M:%S:%H',
+          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+          'with_priority' => true,
+          'message_format' => 'auto',
+        )
+        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 6), record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "11111", record["pid"]
+          assert_equal "ID24224", record["msgid"]
+          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                       record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+        text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+          assert_equal(@expected.merge('pri' => 16), record)
+        end
+        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+        @parser.parse(text) do |time, record|
+          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+          assert_equal "-", record["pid"]
+          assert_equal "-", record["msgid"]
+          assert_equal "-", record["extradata"]
+          assert_equal "Hi, from Fluentd!", record["message"]
+        end
+      end
+    end
   end
 
   class JsonParserTest < ::Test::Unit::TestCase
@@ -572,11 +756,10 @@ module ParserTest
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern
+    def test_parse_with_null_value_pattern(param)
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
@@ -590,11 +773,10 @@ module ParserTest
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string
+    def test_parse_with_null_empty_string(param)
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse("\t ") do |time, record|
@@ -661,29 +843,27 @@ module ParserTest
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern
+    def test_parse_with_null_value_pattern(param)
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-,null,NULL,,--,nuLL") do |time, record|
         assert_nil record['a']
         assert_nil record['b']
         assert_nil record['c']
-        assert_equal record['d'], ''
+        assert_nil record['d']
         assert_equal record['e'], '--'
         assert_equal record['f'], 'nuLL'
       end
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string
+    def test_parse_with_null_empty_string(param)
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
-        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse(", ") do |time, record|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.32
+  version: 0.12.33
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-03 00:00:00.000000000 Z
+date: 2017-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -425,6 +425,7 @@ files:
 - test/plugin/data/2010/01/20100102.log
 - test/plugin/data/log/bar
 - test/plugin/data/log/foo/bar.log
+- test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
 - test/plugin/test_buf_file.rb
 - test/plugin/test_buf_memory.rb
@@ -515,6 +516,7 @@ test_files:
 - test/plugin/data/2010/01/20100102.log
 - test/plugin/data/log/bar
 - test/plugin/data/log/foo/bar.log
+- test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
 - test/plugin/test_buf_file.rb
 - test/plugin/test_buf_memory.rb